dd-trace 5.3.0 → 5.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/package.json +2 -2
  2. package/packages/datadog-instrumentations/src/jest.js +15 -14
  3. package/packages/datadog-instrumentations/src/mocha.js +139 -13
  4. package/packages/datadog-plugin-cypress/src/plugin.js +27 -11
  5. package/packages/datadog-plugin-jest/src/index.js +4 -8
  6. package/packages/datadog-plugin-kafkajs/src/consumer.js +16 -0
  7. package/packages/datadog-plugin-mocha/src/index.js +38 -17
  8. package/packages/dd-trace/src/appsec/iast/context/context-plugin.js +90 -0
  9. package/packages/dd-trace/src/appsec/iast/context/kafka-ctx-plugin.js +14 -0
  10. package/packages/dd-trace/src/appsec/iast/iast-plugin.js +8 -0
  11. package/packages/dd-trace/src/appsec/iast/index.js +4 -4
  12. package/packages/dd-trace/src/appsec/iast/overhead-controller.js +1 -1
  13. package/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js +1 -0
  14. package/packages/dd-trace/src/appsec/iast/taint-tracking/index.js +10 -0
  15. package/packages/dd-trace/src/appsec/iast/taint-tracking/operations-taint-object.js +53 -0
  16. package/packages/dd-trace/src/appsec/iast/taint-tracking/operations.js +10 -46
  17. package/packages/dd-trace/src/appsec/iast/taint-tracking/plugin.js +13 -9
  18. package/packages/dd-trace/src/appsec/iast/taint-tracking/plugins/kafka.js +47 -0
  19. package/packages/dd-trace/src/appsec/iast/taint-tracking/source-types.js +3 -1
  20. package/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js +29 -2
  21. package/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js +1 -1
  22. package/packages/dd-trace/src/appsec/remote_config/capabilities.js +2 -1
  23. package/packages/dd-trace/src/appsec/remote_config/index.js +1 -0
  24. package/packages/dd-trace/src/plugins/ci_plugin.js +1 -1
  25. package/packages/dd-trace/src/plugins/util/test.js +17 -1
  26. package/packages/dd-trace/src/telemetry/index.js +3 -0
  27. package/packages/dd-trace/src/telemetry/logs/index.js +2 -2
  28. package/packages/dd-trace/src/telemetry/send-data.js +0 -3
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dd-trace",
3
- "version": "5.3.0",
3
+ "version": "5.4.0",
4
4
  "description": "Datadog APM tracing client for JavaScript",
5
5
  "main": "index.js",
6
6
  "typings": "index.d.ts",
@@ -71,7 +71,7 @@
71
71
  "dependencies": {
72
72
  "@datadog/native-appsec": "7.0.0",
73
73
  "@datadog/native-iast-rewriter": "2.2.3",
74
- "@datadog/native-iast-taint-tracking": "1.6.4",
74
+ "@datadog/native-iast-taint-tracking": "1.7.0",
75
75
  "@datadog/native-metrics": "^2.0.0",
76
76
  "@datadog/pprof": "5.0.0",
77
77
  "@datadog/sketches-js": "^2.1.0",
@@ -9,7 +9,9 @@ const {
9
9
  JEST_WORKER_COVERAGE_PAYLOAD_CODE,
10
10
  getTestLineStart,
11
11
  getTestSuitePath,
12
- getTestParametersString
12
+ getTestParametersString,
13
+ EFD_STRING,
14
+ removeEfdStringFromTestName
13
15
  } = require('../../dd-trace/src/plugins/util/test')
14
16
  const {
15
17
  getFormattedJestTestParameters,
@@ -57,9 +59,6 @@ let hasForcedToRunSuites = false
57
59
  let isEarlyFlakeDetectionEnabled = false
58
60
  let earlyFlakeDetectionNumRetries = 0
59
61
 
60
- const EFD_STRING = "Retried by Datadog's Early Flake Detection"
61
- const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g')
62
-
63
62
  const sessionAsyncResource = new AsyncResource('bound-anonymous-fn')
64
63
 
65
64
  const specStatusToTestStatus = {
@@ -105,10 +104,6 @@ function getEfdTestName (testName, numAttempt) {
105
104
  return `${EFD_STRING} (#${numAttempt}): ${testName}`
106
105
  }
107
106
 
108
- function removeEfdTestName (testName) {
109
- return testName.replace(EFD_TEST_NAME_REGEX, '')
110
- }
111
-
112
107
  function getWrappedEnvironment (BaseEnvironment, jestVersion) {
113
108
  return class DatadogEnvironment extends BaseEnvironment {
114
109
  constructor (config, context) {
@@ -116,12 +111,17 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
116
111
  const rootDir = config.globalConfig ? config.globalConfig.rootDir : config.rootDir
117
112
  this.rootDir = rootDir
118
113
  this.testSuite = getTestSuitePath(context.testPath, rootDir)
119
- this.testFileAbsolutePath = context.testPath
120
114
  this.nameToParams = {}
121
115
  this.global._ddtrace = global._ddtrace
122
116
 
123
117
  this.testEnvironmentOptions = getTestEnvironmentOptions(config)
124
118
 
119
+ const repositoryRoot = this.testEnvironmentOptions._ddRepositoryRoot
120
+
121
+ if (repositoryRoot) {
122
+ this.testSourceFile = getTestSuitePath(context.testPath, repositoryRoot)
123
+ }
124
+
125
125
  this.isEarlyFlakeDetectionEnabled = this.testEnvironmentOptions._ddIsEarlyFlakeDetectionEnabled
126
126
 
127
127
  if (this.isEarlyFlakeDetectionEnabled) {
@@ -152,7 +152,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
152
152
  // we use its describe block to get the full name
153
153
  getTestNameFromAddTestEvent (event, state) {
154
154
  const describeSuffix = getJestTestName(state.currentDescribeBlock)
155
- return removeEfdTestName(`${describeSuffix} ${event.testName}`).trim()
155
+ return removeEfdStringFromTestName(`${describeSuffix} ${event.testName}`).trim()
156
156
  }
157
157
 
158
158
  async handleTestEvent (event, state) {
@@ -186,7 +186,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
186
186
  const testName = getJestTestName(event.test)
187
187
 
188
188
  if (this.isEarlyFlakeDetectionEnabled) {
189
- const originalTestName = removeEfdTestName(testName)
189
+ const originalTestName = removeEfdStringFromTestName(testName)
190
190
  isNewTest = retriedTestsToNumAttempts.has(originalTestName)
191
191
  if (isNewTest) {
192
192
  numEfdRetry = retriedTestsToNumAttempts.get(originalTestName)
@@ -196,9 +196,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
196
196
 
197
197
  asyncResource.runInAsyncScope(() => {
198
198
  testStartCh.publish({
199
- name: removeEfdTestName(testName),
199
+ name: removeEfdStringFromTestName(testName),
200
200
  suite: this.testSuite,
201
- testFileAbsolutePath: this.testFileAbsolutePath,
201
+ testSourceFile: this.testSourceFile,
202
202
  runner: 'jest-circus',
203
203
  testParameters,
204
204
  frameworkVersion: jestVersion,
@@ -249,7 +249,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
249
249
  testSkippedCh.publish({
250
250
  name: getJestTestName(event.test),
251
251
  suite: this.testSuite,
252
- testFileAbsolutePath: this.testFileAbsolutePath,
252
+ testSourceFile: this.testSourceFile,
253
253
  runner: 'jest-circus',
254
254
  frameworkVersion: jestVersion,
255
255
  testStartLine: getTestLineStart(event.test.asyncError, this.testSuite)
@@ -635,6 +635,7 @@ addHook({
635
635
  _ddKnownTests,
636
636
  _ddIsEarlyFlakeDetectionEnabled,
637
637
  _ddEarlyFlakeDetectionNumRetries,
638
+ _ddRepositoryRoot,
638
639
  ...restOfTestEnvironmentOptions
639
640
  } = testEnvironmentOptions
640
641
 
@@ -11,7 +11,9 @@ const {
11
11
  mergeCoverage,
12
12
  getTestSuitePath,
13
13
  fromCoverageMapToCoverage,
14
- getCallSites
14
+ getCallSites,
15
+ addEfdStringToTestName,
16
+ removeEfdStringFromTestName
15
17
  } = require('../../dd-trace/src/plugins/util/test')
16
18
 
17
19
  const testStartCh = channel('ci:mocha:test:start')
@@ -21,6 +23,7 @@ const testFinishCh = channel('ci:mocha:test:finish')
21
23
  const parameterizedTestCh = channel('ci:mocha:test:parameterize')
22
24
 
23
25
  const libraryConfigurationCh = channel('ci:mocha:library-configuration')
26
+ const knownTestsCh = channel('ci:mocha:known-tests')
24
27
  const skippableSuitesCh = channel('ci:mocha:test-suite:skippable')
25
28
 
26
29
  const testSessionStartCh = channel('ci:mocha:session:start')
@@ -40,6 +43,7 @@ const testToAr = new WeakMap()
40
43
  const originalFns = new WeakMap()
41
44
  const testFileToSuiteAr = new Map()
42
45
  const testToStartLine = new WeakMap()
46
+ const newTests = {}
43
47
 
44
48
  // `isWorker` is true if it's a Mocha worker
45
49
  let isWorker = false
@@ -54,6 +58,10 @@ let skippedSuites = []
54
58
  const unskippableSuites = []
55
59
  let isForcedToRun = false
56
60
  let itrCorrelationId = ''
61
+ let isEarlyFlakeDetectionEnabled = false
62
+ let earlyFlakeDetectionNumRetries = 0
63
+ let isSuitesSkippingEnabled = false
64
+ let knownTests = []
57
65
 
58
66
  function getSuitesByTestFile (root) {
59
67
  const suitesByTestFile = {}
@@ -93,6 +101,26 @@ function isRetry (test) {
93
101
  return test._currentRetry !== undefined && test._currentRetry !== 0
94
102
  }
95
103
 
104
+ function getTestFullName (test) {
105
+ return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}`
106
+ }
107
+
108
+ function isNewTest (test) {
109
+ return !knownTests.includes(getTestFullName(test))
110
+ }
111
+
112
+ function retryTest (test) {
113
+ const originalTestName = test.title
114
+ const suite = test.parent
115
+ for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) {
116
+ const clonedTest = test.clone()
117
+ clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1)
118
+ suite.addTest(clonedTest)
119
+ clonedTest._ddIsNew = true
120
+ clonedTest._ddIsEfdRetry = true
121
+ }
122
+ }
123
+
96
124
  function getTestAsyncResource (test) {
97
125
  if (!test.fn) {
98
126
  return testToAr.get(test)
@@ -123,6 +151,19 @@ function mochaHook (Runner) {
123
151
 
124
152
  patched.add(Runner)
125
153
 
154
+ shimmer.wrap(Runner.prototype, 'runTests', runTests => function (suite, fn) {
155
+ if (isEarlyFlakeDetectionEnabled) {
156
+ // by the time we reach `this.on('test')`, it is too late. We need to add retries here
157
+ suite.tests.forEach(test => {
158
+ if (!test.isPending() && isNewTest(test)) {
159
+ test._ddIsNew = true
160
+ retryTest(test)
161
+ }
162
+ })
163
+ }
164
+ return runTests.apply(this, arguments)
165
+ })
166
+
126
167
  shimmer.wrap(Runner.prototype, 'run', run => function () {
127
168
  if (!testStartCh.hasSubscribers || isWorker) {
128
169
  return run.apply(this, arguments)
@@ -144,6 +185,24 @@ function mochaHook (Runner) {
144
185
  status = 'fail'
145
186
  }
146
187
 
188
+ if (isEarlyFlakeDetectionEnabled) {
189
+ /**
190
+ * If Early Flake Detection (EFD) is enabled the logic is as follows:
191
+ * - If all attempts for a test are failing, the test has failed and we will let the test process fail.
192
+ * - If just a single attempt passes, we will prevent the test process from failing.
193
+ * The rationale behind is the following: you may still be able to block your CI pipeline by gating
194
+ * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too.
195
+ */
196
+ for (const tests of Object.values(newTests)) {
197
+ const failingNewTests = tests.filter(test => test.isFailed())
198
+ const areAllNewTestsFailing = failingNewTests.length === tests.length
199
+ if (failingNewTests.length && !areAllNewTestsFailing) {
200
+ this.stats.failures -= failingNewTests.length
201
+ this.failures -= failingNewTests.length
202
+ }
203
+ }
204
+ }
205
+
147
206
  if (status === 'fail') {
148
207
  error = new Error(`Failed tests: ${this.failures}.`)
149
208
  }
@@ -168,7 +227,8 @@ function mochaHook (Runner) {
168
227
  numSkippedSuites: skippedSuites.length,
169
228
  hasForcedToRunSuites: isForcedToRun,
170
229
  hasUnskippableSuites: !!unskippableSuites.length,
171
- error
230
+ error,
231
+ isEarlyFlakeDetectionEnabled
172
232
  })
173
233
  }))
174
234
 
@@ -253,8 +313,35 @@ function mochaHook (Runner) {
253
313
  const testStartLine = testToStartLine.get(test)
254
314
  const asyncResource = new AsyncResource('bound-anonymous-fn')
255
315
  testToAr.set(test.fn, asyncResource)
316
+
317
+ const {
318
+ file: testSuiteAbsolutePath,
319
+ title,
320
+ _ddIsNew: isNew,
321
+ _ddIsEfdRetry: isEfdRetry
322
+ } = test
323
+
324
+ const testInfo = {
325
+ testName: test.fullTitle(),
326
+ testSuiteAbsolutePath,
327
+ title,
328
+ isNew,
329
+ isEfdRetry,
330
+ testStartLine
331
+ }
332
+
333
+ // We want to store the result of the new tests
334
+ if (isNew) {
335
+ const testFullName = getTestFullName(test)
336
+ if (newTests[testFullName]) {
337
+ newTests[testFullName].push(test)
338
+ } else {
339
+ newTests[testFullName] = [test]
340
+ }
341
+ }
342
+
256
343
  asyncResource.runInAsyncScope(() => {
257
- testStartCh.publish({ test, testStartLine })
344
+ testStartCh.publish(testInfo)
258
345
  })
259
346
  })
260
347
 
@@ -323,10 +410,23 @@ function mochaHook (Runner) {
323
410
  })
324
411
 
325
412
  this.on('pending', (test) => {
413
+ const testStartLine = testToStartLine.get(test)
414
+ const {
415
+ file: testSuiteAbsolutePath,
416
+ title
417
+ } = test
418
+
419
+ const testInfo = {
420
+ testName: test.fullTitle(),
421
+ testSuiteAbsolutePath,
422
+ title,
423
+ testStartLine
424
+ }
425
+
326
426
  const asyncResource = getTestAsyncResource(test)
327
427
  if (asyncResource) {
328
428
  asyncResource.runInAsyncScope(() => {
329
- skipCh.publish(test)
429
+ skipCh.publish(testInfo)
330
430
  })
331
431
  } else {
332
432
  // if there is no async resource, the test has been skipped through `test.skip`
@@ -338,7 +438,7 @@ function mochaHook (Runner) {
338
438
  testToAr.set(test, skippedTestAsyncResource)
339
439
  }
340
440
  skippedTestAsyncResource.runInAsyncScope(() => {
341
- skipCh.publish(test)
441
+ skipCh.publish(testInfo)
342
442
  })
343
443
  }
344
444
  })
@@ -358,8 +458,8 @@ function mochaEachHook (mochaEach) {
358
458
  const [params] = arguments
359
459
  const { it, ...rest } = mochaEach.apply(this, arguments)
360
460
  return {
361
- it: function (name) {
362
- parameterizedTestCh.publish({ name, params })
461
+ it: function (title) {
462
+ parameterizedTestCh.publish({ title, params })
363
463
  it.apply(this, arguments)
364
464
  },
365
465
  ...rest
@@ -425,17 +525,43 @@ addHook({
425
525
  global.run()
426
526
  }
427
527
 
428
- const onReceivedConfiguration = ({ err }) => {
528
+ const onReceivedKnownTests = ({ err, knownTests: receivedKnownTests }) => {
429
529
  if (err) {
430
- return global.run()
530
+ knownTests = []
531
+ isEarlyFlakeDetectionEnabled = false
532
+ } else {
533
+ knownTests = receivedKnownTests
534
+ }
535
+
536
+ if (isSuitesSkippingEnabled) {
537
+ skippableSuitesCh.publish({
538
+ onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites)
539
+ })
540
+ } else {
541
+ global.run()
431
542
  }
432
- if (!skippableSuitesCh.hasSubscribers) {
543
+ }
544
+
545
+ const onReceivedConfiguration = ({ err, libraryConfig }) => {
546
+ if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) {
433
547
  return global.run()
434
548
  }
435
549
 
436
- skippableSuitesCh.publish({
437
- onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites)
438
- })
550
+ isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
551
+ isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled
552
+ earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
553
+
554
+ if (isEarlyFlakeDetectionEnabled) {
555
+ knownTestsCh.publish({
556
+ onDone: mochaRunAsyncResource.bind(onReceivedKnownTests)
557
+ })
558
+ } else if (isSuitesSkippingEnabled) {
559
+ skippableSuitesCh.publish({
560
+ onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites)
561
+ })
562
+ } else {
563
+ global.run()
564
+ }
439
565
  }
440
566
 
441
567
  mochaRunAsyncResource.runInAsyncScope(() => {
@@ -115,7 +115,8 @@ function getSuiteStatus (suiteStats) {
115
115
  if (suiteStats.failures !== undefined && suiteStats.failures > 0) {
116
116
  return 'fail'
117
117
  }
118
- if (suiteStats.tests !== undefined && suiteStats.tests === suiteStats.pending) {
118
+ if (suiteStats.tests !== undefined &&
119
+ (suiteStats.tests === suiteStats.pending || suiteStats.tests === suiteStats.skipped)) {
119
120
  return 'skip'
120
121
  }
121
122
  return 'pass'
@@ -246,6 +247,10 @@ module.exports = (on, config) => {
246
247
  if (testSessionSpan && testModuleSpan) {
247
248
  testSuiteTags[TEST_SESSION_ID] = testSessionSpan.context().toTraceId()
248
249
  testSuiteTags[TEST_MODULE_ID] = testModuleSpan.context().toSpanId()
250
+ // If testSuiteSpan couldn't be created, we'll use the testModuleSpan as the parent
251
+ if (!testSuiteSpan) {
252
+ testSuiteTags[TEST_SUITE_ID] = testModuleSpan.context().toSpanId()
253
+ }
249
254
  }
250
255
 
251
256
  const {
@@ -286,6 +291,19 @@ module.exports = (on, config) => {
286
291
  })
287
292
  }
288
293
 
294
+ function getTestSuiteSpan (suite) {
295
+ const testSuiteSpanMetadata = getTestSuiteCommonTags(command, frameworkVersion, suite, TEST_FRAMEWORK_NAME)
296
+ ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
297
+ return tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_suite`, {
298
+ childOf: testModuleSpan,
299
+ tags: {
300
+ [COMPONENT]: TEST_FRAMEWORK_NAME,
301
+ ...testEnvironmentMetadata,
302
+ ...testSuiteSpanMetadata
303
+ }
304
+ })
305
+ }
306
+
289
307
  on('before:run', (details) => {
290
308
  return getLibraryConfiguration(tracer, testConfiguration).then(({ err, libraryConfig }) => {
291
309
  if (err) {
@@ -349,6 +367,13 @@ module.exports = (on, config) => {
349
367
  const cypressTests = tests || []
350
368
  const finishedTests = finishedTestsByFile[spec.relative] || []
351
369
 
370
+ if (!testSuiteSpan) {
371
+ // dd:testSuiteStart hasn't been triggered for whatever reason
372
+ // We will create the test suite span on the spot if that's the case
373
+ log.warn('There was an error creating the test suite event.')
374
+ testSuiteSpan = getTestSuiteSpan(spec.relative)
375
+ }
376
+
352
377
  // Get tests that didn't go through `dd:afterEach`
353
378
  // and create a skipped test span for each of them
354
379
  cypressTests.filter(({ title }) => {
@@ -470,16 +495,7 @@ module.exports = (on, config) => {
470
495
  if (testSuiteSpan) {
471
496
  return null
472
497
  }
473
- const testSuiteSpanMetadata = getTestSuiteCommonTags(command, frameworkVersion, suite, TEST_FRAMEWORK_NAME)
474
- testSuiteSpan = tracer.startSpan(`${TEST_FRAMEWORK_NAME}.test_suite`, {
475
- childOf: testModuleSpan,
476
- tags: {
477
- [COMPONENT]: TEST_FRAMEWORK_NAME,
478
- ...testEnvironmentMetadata,
479
- ...testSuiteSpanMetadata
480
- }
481
- })
482
- ciVisEvent(TELEMETRY_EVENT_CREATED, 'suite')
498
+ testSuiteSpan = getTestSuiteSpan(suite)
483
499
  return null
484
500
  },
485
501
  'dd:beforeEach': (test) => {
@@ -16,7 +16,6 @@ const {
16
16
  TEST_CODE_OWNERS,
17
17
  ITR_CORRELATION_ID,
18
18
  TEST_SOURCE_FILE,
19
- getTestSuitePath,
20
19
  TEST_IS_NEW,
21
20
  TEST_EARLY_FLAKE_IS_RETRY,
22
21
  TEST_EARLY_FLAKE_IS_ENABLED
@@ -141,6 +140,7 @@ class JestPlugin extends CiPlugin {
141
140
  config._ddItrCorrelationId = this.itrCorrelationId
142
141
  config._ddIsEarlyFlakeDetectionEnabled = !!this.libraryConfig?.isEarlyFlakeDetectionEnabled
143
142
  config._ddEarlyFlakeDetectionNumRetries = this.libraryConfig?.earlyFlakeDetectionNumRetries ?? 0
143
+ config._ddRepositoryRoot = this.repositoryRoot
144
144
  })
145
145
  })
146
146
 
@@ -311,7 +311,7 @@ class JestPlugin extends CiPlugin {
311
311
  testParameters,
312
312
  frameworkVersion,
313
313
  testStartLine,
314
- testFileAbsolutePath,
314
+ testSourceFile,
315
315
  isNew,
316
316
  isEfdRetry
317
317
  } = test
@@ -324,12 +324,8 @@ class JestPlugin extends CiPlugin {
324
324
  if (testStartLine) {
325
325
  extraTags[TEST_SOURCE_START] = testStartLine
326
326
  }
327
- if (testFileAbsolutePath) {
328
- extraTags[TEST_SOURCE_FILE] = getTestSuitePath(testFileAbsolutePath, this.repositoryRoot)
329
- } else {
330
- // If for whatever we don't have the full path, we'll set the source file to the suite name
331
- extraTags[TEST_SOURCE_FILE] = suite
332
- }
327
+ // If for whatever we don't have the source file, we'll fall back to the suite name
328
+ extraTags[TEST_SOURCE_FILE] = testSourceFile || suite
333
329
 
334
330
  if (isNew) {
335
331
  extraTags[TEST_IS_NEW] = 'true'
@@ -1,8 +1,12 @@
1
1
  'use strict'
2
2
 
3
+ const dc = require('dc-polyfill')
3
4
  const { getMessageSize, CONTEXT_PROPAGATION_KEY } = require('../../dd-trace/src/datastreams/processor')
4
5
  const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer')
5
6
 
7
+ const afterStartCh = dc.channel('dd-trace:kafkajs:consumer:afterStart')
8
+ const beforeFinishCh = dc.channel('dd-trace:kafkajs:consumer:beforeFinish')
9
+
6
10
  class KafkajsConsumerPlugin extends ConsumerPlugin {
7
11
  static get id () { return 'kafkajs' }
8
12
  static get operation () { return 'consume' }
@@ -79,6 +83,18 @@ class KafkajsConsumerPlugin extends ConsumerPlugin {
79
83
  this.tracer
80
84
  .setCheckpoint(['direction:in', `group:${groupId}`, `topic:${topic}`, 'type:kafka'], span, payloadSize)
81
85
  }
86
+
87
+ if (afterStartCh.hasSubscribers) {
88
+ afterStartCh.publish({ topic, partition, message, groupId })
89
+ }
90
+ }
91
+
92
+ finish () {
93
+ if (beforeFinishCh.hasSubscribers) {
94
+ beforeFinishCh.publish()
95
+ }
96
+
97
+ super.finish()
82
98
  }
83
99
  }
84
100
 
@@ -16,7 +16,11 @@ const {
16
16
  TEST_ITR_FORCED_RUN,
17
17
  TEST_CODE_OWNERS,
18
18
  ITR_CORRELATION_ID,
19
- TEST_SOURCE_FILE
19
+ TEST_SOURCE_FILE,
20
+ removeEfdStringFromTestName,
21
+ TEST_IS_NEW,
22
+ TEST_EARLY_FLAKE_IS_RETRY,
23
+ TEST_EARLY_FLAKE_IS_ENABLED
20
24
  } = require('../../dd-trace/src/plugins/util/test')
21
25
  const { COMPONENT } = require('../../dd-trace/src/constants')
22
26
  const {
@@ -39,7 +43,7 @@ class MochaPlugin extends CiPlugin {
39
43
  super(...args)
40
44
 
41
45
  this._testSuites = new Map()
42
- this._testNameToParams = {}
46
+ this._testTitleToParams = {}
43
47
  this.sourceRoot = process.cwd()
44
48
 
45
49
  this.addSub('ci:mocha:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => {
@@ -131,9 +135,9 @@ class MochaPlugin extends CiPlugin {
131
135
  }
132
136
  })
133
137
 
134
- this.addSub('ci:mocha:test:start', ({ test, testStartLine }) => {
138
+ this.addSub('ci:mocha:test:start', (testInfo) => {
135
139
  const store = storage.getStore()
136
- const span = this.startTestSpan(test, testStartLine)
140
+ const span = this.startTestSpan(testInfo)
137
141
 
138
142
  this.enter(span, store)
139
143
  })
@@ -156,12 +160,12 @@ class MochaPlugin extends CiPlugin {
156
160
  }
157
161
  })
158
162
 
159
- this.addSub('ci:mocha:test:skip', (test) => {
163
+ this.addSub('ci:mocha:test:skip', (testInfo) => {
160
164
  const store = storage.getStore()
161
165
  // skipped through it.skip, so the span is not created yet
162
166
  // for this test
163
167
  if (!store) {
164
- const testSpan = this.startTestSpan(test)
168
+ const testSpan = this.startTestSpan(testInfo)
165
169
  this.enter(testSpan, store)
166
170
  }
167
171
  })
@@ -179,8 +183,8 @@ class MochaPlugin extends CiPlugin {
179
183
  }
180
184
  })
181
185
 
182
- this.addSub('ci:mocha:test:parameterize', ({ name, params }) => {
183
- this._testNameToParams[name] = params
186
+ this.addSub('ci:mocha:test:parameterize', ({ title, params }) => {
187
+ this._testTitleToParams[title] = params
184
188
  })
185
189
 
186
190
  this.addSub('ci:mocha:session:finish', ({
@@ -190,7 +194,8 @@ class MochaPlugin extends CiPlugin {
190
194
  numSkippedSuites,
191
195
  hasForcedToRunSuites,
192
196
  hasUnskippableSuites,
193
- error
197
+ error,
198
+ isEarlyFlakeDetectionEnabled
194
199
  }) => {
195
200
  if (this.testSessionSpan) {
196
201
  const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {}
@@ -217,6 +222,10 @@ class MochaPlugin extends CiPlugin {
217
222
  }
218
223
  )
219
224
 
225
+ if (isEarlyFlakeDetectionEnabled) {
226
+ this.testSessionSpan.setTag(TEST_EARLY_FLAKE_IS_ENABLED, 'true')
227
+ }
228
+
220
229
  this.testModuleSpan.finish()
221
230
  this.telemetry.ciVisEvent(TELEMETRY_EVENT_FINISHED, 'module')
222
231
  this.testSessionSpan.finish()
@@ -228,12 +237,19 @@ class MochaPlugin extends CiPlugin {
228
237
  })
229
238
  }
230
239
 
231
- startTestSpan (test, testStartLine) {
232
- const testName = test.fullTitle()
233
- const { file: testSuiteAbsolutePath, title } = test
240
+ startTestSpan (testInfo) {
241
+ const {
242
+ testSuiteAbsolutePath,
243
+ title,
244
+ isNew,
245
+ isEfdRetry,
246
+ testStartLine
247
+ } = testInfo
248
+
249
+ const testName = removeEfdStringFromTestName(testInfo.testName)
234
250
 
235
251
  const extraTags = {}
236
- const testParametersString = getTestParametersString(this._testNameToParams, title)
252
+ const testParametersString = getTestParametersString(this._testTitleToParams, title)
237
253
  if (testParametersString) {
238
254
  extraTags[TEST_PARAMETERS] = testParametersString
239
255
  }
@@ -245,14 +261,19 @@ class MochaPlugin extends CiPlugin {
245
261
  const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot)
246
262
  const testSuiteSpan = this._testSuites.get(testSuiteAbsolutePath)
247
263
 
248
- const testSourceFile = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
249
-
250
- if (testSourceFile) {
251
- extraTags[TEST_SOURCE_FILE] = testSourceFile
264
+ if (this.repositoryRoot !== this.sourceRoot && !!this.repositoryRoot) {
265
+ extraTags[TEST_SOURCE_FILE] = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot)
252
266
  } else {
253
267
  extraTags[TEST_SOURCE_FILE] = testSuite
254
268
  }
255
269
 
270
+ if (isNew) {
271
+ extraTags[TEST_IS_NEW] = 'true'
272
+ if (isEfdRetry) {
273
+ extraTags[TEST_EARLY_FLAKE_IS_RETRY] = 'true'
274
+ }
275
+ }
276
+
256
277
  return super.startTestSpan(testName, testSuite, testSuiteSpan, extraTags)
257
278
  }
258
279
  }
@@ -0,0 +1,90 @@
1
+ 'use strict'
2
+
3
+ const { storage } = require('../../../../../datadog-core')
4
+ const iastContextFunctions = require('../iast-context')
5
+ const overheadController = require('../overhead-controller')
6
+ const { IastPlugin } = require('../iast-plugin')
7
+ const { IAST_ENABLED_TAG_KEY } = require('../tags')
8
+ const { createTransaction, removeTransaction } = require('../taint-tracking/operations')
9
+ const vulnerabilityReporter = require('../vulnerability-reporter')
10
+ const { TagKey } = require('../telemetry/iast-metric')
11
+
12
+ class IastContextPlugin extends IastPlugin {
13
+ startCtxOn (channelName, tag) {
14
+ super.addSub(channelName, (message) => this.startContext())
15
+
16
+ this._getAndRegisterSubscription({
17
+ channelName,
18
+ tag,
19
+ tagKey: TagKey.SOURCE_TYPE
20
+ })
21
+ }
22
+
23
+ finishCtxOn (channelName) {
24
+ super.addSub(channelName, (message) => this.finishContext())
25
+ }
26
+
27
+ getRootSpan (store) {
28
+ return store?.span
29
+ }
30
+
31
+ getTopContext () {
32
+ return {}
33
+ }
34
+
35
+ newIastContext (rootSpan) {
36
+ return { rootSpan }
37
+ }
38
+
39
+ addIastEnabledTag (isRequestAcquired, rootSpan) {
40
+ if (rootSpan?.addTags) {
41
+ rootSpan.addTags({
42
+ [IAST_ENABLED_TAG_KEY]: isRequestAcquired ? 1 : 0
43
+ })
44
+ }
45
+ }
46
+
47
+ startContext () {
48
+ let isRequestAcquired = false
49
+ let iastContext
50
+
51
+ const store = storage.getStore()
52
+ if (store) {
53
+ const topContext = this.getTopContext()
54
+ const rootSpan = this.getRootSpan(store)
55
+
56
+ isRequestAcquired = overheadController.acquireRequest(rootSpan)
57
+ if (isRequestAcquired) {
58
+ iastContext = iastContextFunctions.saveIastContext(store, topContext, this.newIastContext(rootSpan))
59
+ createTransaction(rootSpan.context().toSpanId(), iastContext)
60
+ overheadController.initializeRequestContext(iastContext)
61
+ }
62
+ this.addIastEnabledTag(isRequestAcquired, rootSpan)
63
+ }
64
+
65
+ return {
66
+ isRequestAcquired,
67
+ iastContext,
68
+ store
69
+ }
70
+ }
71
+
72
+ finishContext () {
73
+ const store = storage.getStore()
74
+ if (store) {
75
+ const topContext = this.getTopContext()
76
+ const iastContext = iastContextFunctions.getIastContext(store, topContext)
77
+ const rootSpan = iastContext?.rootSpan
78
+ if (iastContext && rootSpan) {
79
+ vulnerabilityReporter.sendVulnerabilities(iastContext.vulnerabilities, rootSpan)
80
+ removeTransaction(iastContext)
81
+ }
82
+
83
+ if (iastContextFunctions.cleanIastContext(store, topContext, iastContext)) {
84
+ overheadController.releaseRequest()
85
+ }
86
+ }
87
+ }
88
+ }
89
+
90
+ module.exports = IastContextPlugin
@@ -0,0 +1,14 @@
1
+ 'use strict'
2
+
3
+ const { KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE } = require('../taint-tracking/source-types')
4
+ const IastContextPlugin = require('./context-plugin')
5
+
6
+ class KafkaContextPlugin extends IastContextPlugin {
7
+ onConfigure () {
8
+ this.startCtxOn('dd-trace:kafkajs:consumer:afterStart', [KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE])
9
+
10
+ this.finishCtxOn('dd-trace:kafkajs:consumer:beforeFinish')
11
+ }
12
+ }
13
+
14
+ module.exports = new KafkaContextPlugin()
@@ -101,6 +101,14 @@ class IastPlugin extends Plugin {
101
101
  }
102
102
  }
103
103
 
104
+ enable () {
105
+ this.configure(true)
106
+ }
107
+
108
+ disable () {
109
+ this.configure(false)
110
+ }
111
+
104
112
  onConfigure () {}
105
113
 
106
114
  configure (config) {
@@ -22,7 +22,7 @@ const requestClose = dc.channel('dd-trace:incomingHttpRequestEnd')
22
22
  const iastResponseEnd = dc.channel('datadog:iast:response-end')
23
23
 
24
24
  function enable (config, _tracer) {
25
- iastTelemetry.configure(config, config.iast && config.iast.telemetryVerbosity)
25
+ iastTelemetry.configure(config, config.iast?.telemetryVerbosity)
26
26
  enableAllAnalyzers(config)
27
27
  enableTaintTracking(config.iast, iastTelemetry.verbosity)
28
28
  requestStart.subscribe(onIncomingHttpRequestStart)
@@ -43,7 +43,7 @@ function disable () {
43
43
  }
44
44
 
45
45
  function onIncomingHttpRequestStart (data) {
46
- if (data && data.req) {
46
+ if (data?.req) {
47
47
  const store = storage.getStore()
48
48
  if (store) {
49
49
  const topContext = web.getContext(data.req)
@@ -68,11 +68,11 @@ function onIncomingHttpRequestStart (data) {
68
68
  }
69
69
 
70
70
  function onIncomingHttpRequestEnd (data) {
71
- if (data && data.req) {
71
+ if (data?.req) {
72
72
  const store = storage.getStore()
73
73
  const topContext = web.getContext(data.req)
74
74
  const iastContext = iastContextFunctions.getIastContext(store, topContext)
75
- if (iastContext && iastContext.rootSpan) {
75
+ if (iastContext?.rootSpan) {
76
76
  iastResponseEnd.publish(data)
77
77
 
78
78
  const vulnerabilities = iastContext.vulnerabilities
@@ -52,7 +52,7 @@ function _resetGlobalContext () {
52
52
  }
53
53
 
54
54
  function acquireRequest (rootSpan) {
55
- if (availableRequest > 0) {
55
+ if (availableRequest > 0 && rootSpan) {
56
56
  const sampling = config && typeof config.requestSampling === 'number'
57
57
  ? config.requestSampling : 30
58
58
  if (rootSpan.context().toSpanId().slice(-2) <= sampling) {
@@ -2,6 +2,7 @@
2
2
 
3
3
  const csiMethods = [
4
4
  { src: 'concat' },
5
+ { src: 'parse' },
5
6
  { src: 'plusOperator', operator: true },
6
7
  { src: 'random' },
7
8
  { src: 'replace' },
@@ -10,18 +10,28 @@ const {
10
10
  } = require('./operations')
11
11
 
12
12
  const taintTrackingPlugin = require('./plugin')
13
+ const kafkaConsumerPlugin = require('./plugins/kafka')
14
+
15
+ const kafkaContextPlugin = require('../context/kafka-ctx-plugin')
13
16
 
14
17
  module.exports = {
15
18
  enableTaintTracking (config, telemetryVerbosity) {
16
19
  enableRewriter(telemetryVerbosity)
17
20
  enableTaintOperations(telemetryVerbosity)
18
21
  taintTrackingPlugin.enable()
22
+
23
+ kafkaContextPlugin.enable()
24
+ kafkaConsumerPlugin.enable()
25
+
19
26
  setMaxTransactions(config.maxConcurrentRequests)
20
27
  },
21
28
  disableTaintTracking () {
22
29
  disableRewriter()
23
30
  disableTaintOperations()
24
31
  taintTrackingPlugin.disable()
32
+
33
+ kafkaContextPlugin.disable()
34
+ kafkaConsumerPlugin.disable()
25
35
  },
26
36
  setMaxTransactions: setMaxTransactions,
27
37
  createTransaction: createTransaction,
@@ -0,0 +1,53 @@
1
+ 'use strict'
2
+
3
+ const TaintedUtils = require('@datadog/native-iast-taint-tracking')
4
+ const { IAST_TRANSACTION_ID } = require('../iast-context')
5
+ const iastLog = require('../iast-log')
6
+
7
+ function taintObject (iastContext, object, type, keyTainting, keyType) {
8
+ let result = object
9
+ const transactionId = iastContext?.[IAST_TRANSACTION_ID]
10
+ if (transactionId) {
11
+ const queue = [{ parent: null, property: null, value: object }]
12
+ const visited = new WeakSet()
13
+
14
+ while (queue.length > 0) {
15
+ const { parent, property, value, key } = queue.pop()
16
+ if (value === null) {
17
+ continue
18
+ }
19
+
20
+ try {
21
+ if (typeof value === 'string') {
22
+ const tainted = TaintedUtils.newTaintedString(transactionId, value, property, type)
23
+ if (!parent) {
24
+ result = tainted
25
+ } else if (keyTainting && key) {
26
+ const taintedProperty = TaintedUtils.newTaintedString(transactionId, key, property, keyType)
27
+ parent[taintedProperty] = tainted
28
+ } else {
29
+ parent[key] = tainted
30
+ }
31
+ } else if (typeof value === 'object' && !visited.has(value)) {
32
+ visited.add(value)
33
+
34
+ for (const key of Object.keys(value)) {
35
+ queue.push({ parent: value, property: property ? `${property}.${key}` : key, value: value[key], key })
36
+ }
37
+
38
+ if (parent && keyTainting && key) {
39
+ const taintedProperty = TaintedUtils.newTaintedString(transactionId, key, property, keyType)
40
+ parent[taintedProperty] = value
41
+ }
42
+ }
43
+ } catch (e) {
44
+ iastLog.error(`Error visiting property : ${property}`).errorAndPublish(e)
45
+ }
46
+ }
47
+ }
48
+ return result
49
+ }
50
+
51
+ module.exports = {
52
+ taintObject
53
+ }
@@ -2,11 +2,11 @@
2
2
 
3
3
  const TaintedUtils = require('@datadog/native-iast-taint-tracking')
4
4
  const { IAST_TRANSACTION_ID } = require('../iast-context')
5
- const iastLog = require('../iast-log')
6
5
  const iastTelemetry = require('../telemetry')
7
6
  const { REQUEST_TAINTED } = require('../telemetry/iast-metric')
8
7
  const { isInfoAllowed } = require('../telemetry/verbosity')
9
8
  const { getTaintTrackingImpl, getTaintTrackingNoop } = require('./taint-tracking-impl')
9
+ const { taintObject } = require('./operations-taint-object')
10
10
 
11
11
  function createTransaction (id, iastContext) {
12
12
  if (id && iastContext) {
@@ -34,7 +34,7 @@ function removeTransaction (iastContext) {
34
34
  }
35
35
 
36
36
  function newTaintedString (iastContext, string, name, type) {
37
- let result = string
37
+ let result
38
38
  const transactionId = iastContext?.[IAST_TRANSACTION_ID]
39
39
  if (transactionId) {
40
40
  result = TaintedUtils.newTaintedString(transactionId, string, name, type)
@@ -44,56 +44,19 @@ function newTaintedString (iastContext, string, name, type) {
44
44
  return result
45
45
  }
46
46
 
47
- function taintObject (iastContext, object, type, keyTainting, keyType) {
48
- let result = object
47
+ function newTaintedObject (iastContext, obj, name, type) {
48
+ let result
49
49
  const transactionId = iastContext?.[IAST_TRANSACTION_ID]
50
50
  if (transactionId) {
51
- const queue = [{ parent: null, property: null, value: object }]
52
- const visited = new WeakSet()
53
-
54
- while (queue.length > 0) {
55
- const { parent, property, value, key } = queue.pop()
56
- if (value === null) {
57
- continue
58
- }
59
-
60
- try {
61
- if (typeof value === 'string') {
62
- const tainted = TaintedUtils.newTaintedString(transactionId, value, property, type)
63
- if (!parent) {
64
- result = tainted
65
- } else {
66
- if (keyTainting && key) {
67
- const taintedProperty = TaintedUtils.newTaintedString(transactionId, key, property, keyType)
68
- parent[taintedProperty] = tainted
69
- } else {
70
- parent[key] = tainted
71
- }
72
- }
73
- } else if (typeof value === 'object' && !visited.has(value)) {
74
- visited.add(value)
75
-
76
- const keys = Object.keys(value)
77
- for (let i = 0; i < keys.length; i++) {
78
- const key = keys[i]
79
- queue.push({ parent: value, property: property ? `${property}.${key}` : key, value: value[key], key })
80
- }
81
-
82
- if (parent && keyTainting && key) {
83
- const taintedProperty = TaintedUtils.newTaintedString(transactionId, key, property, keyType)
84
- parent[taintedProperty] = value
85
- }
86
- }
87
- } catch (e) {
88
- iastLog.error(`Error visiting property : ${property}`).errorAndPublish(e)
89
- }
90
- }
51
+ result = TaintedUtils.newTaintedObject(transactionId, obj, name, type)
52
+ } else {
53
+ result = obj
91
54
  }
92
55
  return result
93
56
  }
94
57
 
95
58
  function isTainted (iastContext, string) {
96
- let result = false
59
+ let result
97
60
  const transactionId = iastContext?.[IAST_TRANSACTION_ID]
98
61
  if (transactionId) {
99
62
  result = TaintedUtils.isTainted(transactionId, string)
@@ -104,7 +67,7 @@ function isTainted (iastContext, string) {
104
67
  }
105
68
 
106
69
  function getRanges (iastContext, string) {
107
- let result = []
70
+ let result
108
71
  const transactionId = iastContext?.[IAST_TRANSACTION_ID]
109
72
  if (transactionId) {
110
73
  result = TaintedUtils.getRanges(transactionId, string)
@@ -148,6 +111,7 @@ module.exports = {
148
111
  createTransaction,
149
112
  removeTransaction,
150
113
  newTaintedString,
114
+ newTaintedObject,
151
115
  taintObject,
152
116
  isTainted,
153
117
  getRanges,
@@ -3,7 +3,7 @@
3
3
  const { SourceIastPlugin } = require('../iast-plugin')
4
4
  const { getIastContext } = require('../iast-context')
5
5
  const { storage } = require('../../../../../datadog-core')
6
- const { taintObject, newTaintedString } = require('./operations')
6
+ const { taintObject, newTaintedString, getRanges } = require('./operations')
7
7
  const {
8
8
  HTTP_REQUEST_BODY,
9
9
  HTTP_REQUEST_COOKIE_VALUE,
@@ -65,6 +65,18 @@ class TaintTrackingPlugin extends SourceIastPlugin {
65
65
  }
66
66
  )
67
67
 
68
+ this.addSub(
69
+ { channelName: 'apm:graphql:resolve:start', tag: HTTP_REQUEST_BODY },
70
+ (data) => {
71
+ const iastContext = getIastContext(storage.getStore())
72
+ const source = data.context?.source
73
+ const ranges = source && getRanges(iastContext, source)
74
+ if (ranges?.length) {
75
+ this._taintTrackingHandler(ranges[0].iinfo.type, data.args, null, iastContext)
76
+ }
77
+ }
78
+ )
79
+
68
80
  // this is a special case to increment INSTRUMENTED_SOURCE metric for header
69
81
  this.addInstrumentedSource('http', [HTTP_REQUEST_HEADER_VALUE, HTTP_REQUEST_HEADER_NAME])
70
82
  }
@@ -104,14 +116,6 @@ class TaintTrackingPlugin extends SourceIastPlugin {
104
116
  this.taintHeaders(req.headers, iastContext)
105
117
  this.taintUrl(req, iastContext)
106
118
  }
107
-
108
- enable () {
109
- this.configure(true)
110
- }
111
-
112
- disable () {
113
- this.configure(false)
114
- }
115
119
  }
116
120
 
117
121
  module.exports = new TaintTrackingPlugin()
@@ -0,0 +1,47 @@
1
+ 'use strict'
2
+
3
+ const shimmer = require('../../../../../../datadog-shimmer')
4
+ const { storage } = require('../../../../../../datadog-core')
5
+ const { getIastContext } = require('../../iast-context')
6
+ const { KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE } = require('../source-types')
7
+ const { newTaintedObject, newTaintedString } = require('../operations')
8
+ const { SourceIastPlugin } = require('../../iast-plugin')
9
+
10
+ class KafkaConsumerIastPlugin extends SourceIastPlugin {
11
+ onConfigure () {
12
+ this.addSub({ channelName: 'dd-trace:kafkajs:consumer:afterStart', tag: [KAFKA_MESSAGE_KEY, KAFKA_MESSAGE_VALUE] },
13
+ ({ message }) => this.taintKafkaMessage(message)
14
+ )
15
+ }
16
+
17
+ getToStringWrap (toString, iastContext, type) {
18
+ return function () {
19
+ const res = toString.apply(this, arguments)
20
+ return newTaintedString(iastContext, res, undefined, type)
21
+ }
22
+ }
23
+
24
+ taintKafkaMessage (message) {
25
+ const iastContext = getIastContext(storage.getStore())
26
+
27
+ if (iastContext && message) {
28
+ const { key, value } = message
29
+
30
+ if (key && typeof key === 'object') {
31
+ shimmer.wrap(key, 'toString',
32
+ toString => this.getToStringWrap(toString, iastContext, KAFKA_MESSAGE_KEY))
33
+
34
+ newTaintedObject(iastContext, key, undefined, KAFKA_MESSAGE_KEY)
35
+ }
36
+
37
+ if (value && typeof value === 'object') {
38
+ shimmer.wrap(value, 'toString',
39
+ toString => this.getToStringWrap(toString, iastContext, KAFKA_MESSAGE_VALUE))
40
+
41
+ newTaintedObject(iastContext, value, undefined, KAFKA_MESSAGE_VALUE)
42
+ }
43
+ }
44
+ }
45
+ }
46
+
47
+ module.exports = new KafkaConsumerIastPlugin()
@@ -9,5 +9,7 @@ module.exports = {
9
9
  HTTP_REQUEST_PARAMETER: 'http.request.parameter',
10
10
  HTTP_REQUEST_PATH: 'http.request.path',
11
11
  HTTP_REQUEST_PATH_PARAM: 'http.request.path.parameter',
12
- HTTP_REQUEST_URI: 'http.request.uri'
12
+ HTTP_REQUEST_URI: 'http.request.uri',
13
+ KAFKA_MESSAGE_KEY: 'kafka.message.key',
14
+ KAFKA_MESSAGE_VALUE: 'kafka.message.value'
13
15
  }
@@ -7,15 +7,19 @@ const iastContextFunctions = require('../iast-context')
7
7
  const iastLog = require('../iast-log')
8
8
  const { EXECUTED_PROPAGATION } = require('../telemetry/iast-metric')
9
9
  const { isDebugAllowed } = require('../telemetry/verbosity')
10
+ const { taintObject } = require('./operations-taint-object')
10
11
 
11
12
  const mathRandomCallCh = dc.channel('datadog:random:call')
12
13
 
14
+ const JSON_VALUE = 'json.value'
15
+
13
16
  function noop (res) { return res }
14
17
  // NOTE: methods of this object must be synchronized with csi-methods.js file definitions!
15
18
  // Otherwise you may end up rewriting a method and not providing its rewritten implementation
16
19
  const TaintTrackingNoop = {
17
- plusOperator: noop,
18
20
  concat: noop,
21
+ parse: noop,
22
+ plusOperator: noop,
19
23
  random: noop,
20
24
  replace: noop,
21
25
  slice: noop,
@@ -26,7 +30,7 @@ const TaintTrackingNoop = {
26
30
  }
27
31
 
28
32
  function getTransactionId (iastContext) {
29
- return iastContext && iastContext[iastContextFunctions.IAST_TRANSACTION_ID]
33
+ return iastContext?.[iastContextFunctions.IAST_TRANSACTION_ID]
30
34
  }
31
35
 
32
36
  function getContextDefault () {
@@ -120,6 +124,29 @@ function csiMethodsOverrides (getContext) {
120
124
  if (mathRandomCallCh.hasSubscribers) {
121
125
  mathRandomCallCh.publish({ fn })
122
126
  }
127
+ return res
128
+ },
129
+
130
+ parse: function (res, fn, target, json) {
131
+ if (fn === JSON.parse) {
132
+ try {
133
+ const iastContext = getContext()
134
+ const transactionId = getTransactionId(iastContext)
135
+ if (transactionId) {
136
+ const ranges = TaintedUtils.getRanges(transactionId, json)
137
+
138
+ // TODO: first version.
139
+ // here we are losing the original source because taintObject always creates a new tainted
140
+ if (ranges?.length > 0) {
141
+ const range = ranges.find(range => range.iinfo?.type)
142
+ res = taintObject(iastContext, res, range?.iinfo.type || JSON_VALUE)
143
+ }
144
+ }
145
+ } catch (e) {
146
+ iastLog.error(e)
147
+ }
148
+ }
149
+
123
150
  return res
124
151
  }
125
152
  }
@@ -20,7 +20,7 @@ function iterateObject (target, fn, levelKeys = [], depth = 50) {
20
20
 
21
21
  fn(val, nextLevelKeys, target, key)
22
22
 
23
- if (val !== null && typeof val === 'object') {
23
+ if (val !== null && typeof val === 'object' && depth > 0) {
24
24
  iterateObject(val, fn, nextLevelKeys, depth - 1)
25
25
  }
26
26
  })
@@ -14,5 +14,6 @@ module.exports = {
14
14
  APM_TRACING_SAMPLE_RATE: 1n << 12n,
15
15
  APM_TRACING_LOGS_INJECTION: 1n << 13n,
16
16
  APM_TRACING_HTTP_HEADER_TAGS: 1n << 14n,
17
- APM_TRACING_CUSTOM_TAGS: 1n << 15n
17
+ APM_TRACING_CUSTOM_TAGS: 1n << 15n,
18
+ APM_TRACING_ENABLED: 1n << 19n
18
19
  }
@@ -14,6 +14,7 @@ function enable (config) {
14
14
  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_HTTP_HEADER_TAGS, true)
15
15
  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_LOGS_INJECTION, true)
16
16
  rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_SAMPLE_RATE, true)
17
+ rc.updateCapabilities(RemoteConfigCapabilities.APM_TRACING_ENABLED, true)
17
18
 
18
19
  const activation = Activation.fromConfig(config)
19
20
 
@@ -51,7 +51,7 @@ module.exports = class CiPlugin extends Plugin {
51
51
  })
52
52
 
53
53
  this.addSub(`ci:${this.constructor.id}:test-suite:skippable`, ({ onDone }) => {
54
- if (!this.tracer._exporter || !this.tracer._exporter.getSkippableSuites) {
54
+ if (!this.tracer._exporter?.getSkippableSuites) {
55
55
  return onDone({ err: new Error('CI Visibility was not initialized correctly') })
56
56
  }
57
57
  this.tracer._exporter.getSkippableSuites(this.testConfiguration, (err, skippableSuites, itrCorrelationId) => {
@@ -74,6 +74,10 @@ const TEST_CODE_COVERAGE_LINES_PCT = 'test.code_coverage.lines_pct'
74
74
  const JEST_WORKER_TRACE_PAYLOAD_CODE = 60
75
75
  const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61
76
76
 
77
+ // Early flake detection util strings
78
+ const EFD_STRING = "Retried by Datadog's Early Flake Detection"
79
+ const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g')
80
+
77
81
  module.exports = {
78
82
  TEST_CODE_OWNERS,
79
83
  TEST_FRAMEWORK,
@@ -131,7 +135,11 @@ module.exports = {
131
135
  getTestLineStart,
132
136
  getCallSites,
133
137
  removeInvalidMetadata,
134
- parseAnnotations
138
+ parseAnnotations,
139
+ EFD_STRING,
140
+ EFD_TEST_NAME_REGEX,
141
+ removeEfdStringFromTestName,
142
+ addEfdStringToTestName
135
143
  }
136
144
 
137
145
  // Returns pkg manager and its version, separated by '-', e.g. npm-8.15.0 or yarn-1.22.19
@@ -552,3 +560,11 @@ function parseAnnotations (annotations) {
552
560
  return tags
553
561
  }, {})
554
562
  }
563
+
564
+ function addEfdStringToTestName (testName, numAttempt) {
565
+ return `${EFD_STRING} (#${numAttempt}): ${testName}`
566
+ }
567
+
568
+ function removeEfdStringFromTestName (testName) {
569
+ return testName.replace(EFD_TEST_NAME_REGEX, '')
570
+ }
@@ -6,6 +6,7 @@ const dependencies = require('./dependencies')
6
6
  const { sendData } = require('./send-data')
7
7
  const { errors } = require('../startup-log')
8
8
  const { manager: metricsManager } = require('./metrics')
9
+ const logs = require('./logs')
9
10
 
10
11
  const telemetryStartChannel = dc.channel('datadog:telemetry:start')
11
12
  const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
@@ -226,6 +227,7 @@ function createPayload (currReqType, currPayload = {}) {
226
227
  function heartbeat (config, application, host) {
227
228
  heartbeatTimeout = setTimeout(() => {
228
229
  metricsManager.send(config, application, host)
230
+ logs.send(config, application, host)
229
231
 
230
232
  const { reqType, payload } = createPayload('app-heartbeat')
231
233
  sendData(config, application, host, reqType, payload, updateRetryData)
@@ -259,6 +261,7 @@ function start (aConfig, thePluginManager) {
259
261
  integrations = getIntegrations()
260
262
 
261
263
  dependencies.start(config, application, host, getRetryData, updateRetryData)
264
+ logs.start(config)
262
265
 
263
266
  sendData(config, application, host, 'app-started', appStarted(config))
264
267
 
@@ -52,9 +52,9 @@ function stop () {
52
52
  function send (config, application, host) {
53
53
  if (!enabled) return
54
54
 
55
- const logs = { 'logs': logCollector.drain() }
55
+ const logs = logCollector.drain()
56
56
  if (logs) {
57
- sendData(config, application, host, 'logs', logs)
57
+ sendData(config, application, host, 'logs', { logs })
58
58
  }
59
59
  }
60
60
 
@@ -27,9 +27,6 @@ function getAgentlessTelemetryEndpoint (site) {
27
27
  if (site === 'datad0g.com') { // staging
28
28
  return 'https://all-http-intake.logs.datad0g.com'
29
29
  }
30
- if (site === 'datadoghq.eu') {
31
- return 'https://instrumentation-telemetry-intake.eu1.datadoghq.com'
32
- }
33
30
  return `https://instrumentation-telemetry-intake.${site}`
34
31
  }
35
32