datadog-ci 1.41.0 → 1.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +43 -2
- data/README.md +45 -0
- data/lib/datadog/ci/async_writer.rb +1 -4
- data/lib/datadog/ci/concurrent_span.rb +20 -0
- data/lib/datadog/ci/configuration/components.rb +2 -1
- data/lib/datadog/ci/configuration/extensions.rb +2 -0
- data/lib/datadog/ci/configuration/reconfiguration.rb +29 -0
- data/lib/datadog/ci/configuration/settings.rb +9 -2
- data/lib/datadog/ci/contrib/cucumber/formatter.rb +27 -8
- data/lib/datadog/ci/contrib/cucumber/instrumentation.rb +11 -3
- data/lib/datadog/ci/contrib/cuprite/driver.rb +5 -5
- data/lib/datadog/ci/contrib/knapsack/runner.rb +1 -1
- data/lib/datadog/ci/contrib/minitest/helpers.rb +59 -0
- data/lib/datadog/ci/contrib/minitest/reporter.rb +2 -2
- data/lib/datadog/ci/contrib/minitest/runner.rb +14 -1
- data/lib/datadog/ci/contrib/minitest/test.rb +4 -1
- data/lib/datadog/ci/contrib/rspec/example.rb +12 -2
- data/lib/datadog/ci/contrib/rspec/example_group.rb +3 -2
- data/lib/datadog/ci/contrib/rspec/runner.rb +2 -2
- data/lib/datadog/ci/contrib/selenium/navigation.rb +5 -5
- data/lib/datadog/ci/ext/metadata.rb +168 -0
- data/lib/datadog/ci/ext/settings.rb +1 -0
- data/lib/datadog/ci/ext/test.rb +11 -0
- data/lib/datadog/ci/impacted_tests_detection/component.rb +1 -1
- data/lib/datadog/ci/readonly_test_module.rb +16 -0
- data/lib/datadog/ci/readonly_test_session.rb +16 -0
- data/lib/datadog/ci/remote/library_settings_client.rb +1 -1
- data/lib/datadog/ci/remote/slow_test_retries.rb +3 -2
- data/lib/datadog/ci/span.rb +46 -10
- data/lib/datadog/ci/test.rb +75 -18
- data/lib/datadog/ci/test_discovery/component.rb +7 -1
- data/lib/datadog/ci/test_discovery/null_component.rb +4 -0
- data/lib/datadog/ci/test_impact_analysis/component.rb +21 -10
- data/lib/datadog/ci/test_impact_analysis/skippable.rb +1 -1
- data/lib/datadog/ci/test_management/component.rb +4 -4
- data/lib/datadog/ci/test_management/tests_properties.rb +1 -1
- data/lib/datadog/ci/test_retries/component.rb +34 -18
- data/lib/datadog/ci/test_retries/driver/base.rb +2 -2
- data/lib/datadog/ci/test_retries/driver/retry_flake_detection.rb +12 -4
- data/lib/datadog/ci/test_retries/strategy/retry_flake_detection.rb +2 -2
- data/lib/datadog/ci/test_retries/strategy/retry_flaky_fixed.rb +6 -3
- data/lib/datadog/ci/test_suite.rb +11 -0
- data/lib/datadog/ci/test_tracing/component.rb +28 -7
- data/lib/datadog/ci/test_tracing/context.rb +24 -7
- data/lib/datadog/ci/test_tracing/deprecated_total_coverage_metric.rb +1 -1
- data/lib/datadog/ci/test_tracing/known_tests.rb +1 -1
- data/lib/datadog/ci/test_tracing/null_component.rb +10 -1
- data/lib/datadog/ci/test_tracing/transport.rb +3 -22
- data/lib/datadog/ci/transport/adapters/net.rb +24 -4
- data/lib/datadog/ci/transport/adapters/net_http_client.rb +31 -0
- data/lib/datadog/ci/transport/api/evp_proxy.rb +2 -1
- data/lib/datadog/ci/transport/http.rb +20 -4
- data/lib/datadog/ci/utils/protected_tags.rb +27 -0
- data/lib/datadog/ci/utils/rum.rb +1 -1
- data/lib/datadog/ci/version.rb +1 -1
- data/lib/datadog/ci.rb +3 -0
- metadata +4 -1
|
@@ -104,9 +104,9 @@ module Datadog
|
|
|
104
104
|
@test_skipping_enabled = @enabled && remote_configuration.tests_skipping_enabled?
|
|
105
105
|
@code_coverage_enabled = @enabled && remote_configuration.code_coverage_enabled?
|
|
106
106
|
|
|
107
|
-
test_session.
|
|
108
|
-
test_session.
|
|
109
|
-
test_session.
|
|
107
|
+
test_session.set_internal_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_ENABLED, @test_skipping_enabled)
|
|
108
|
+
test_session.set_internal_tag(Ext::Test::TAG_CODE_COVERAGE_ENABLED, @code_coverage_enabled)
|
|
109
|
+
test_session.set_internal_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_TYPE, @test_skipping_mode)
|
|
110
110
|
|
|
111
111
|
if @code_coverage_enabled
|
|
112
112
|
load_datadog_cov!
|
|
@@ -239,6 +239,7 @@ module Datadog
|
|
|
239
239
|
Telemetry.itr_skipped
|
|
240
240
|
|
|
241
241
|
context.incr_tests_skipped_by_tia_count
|
|
242
|
+
test.test_suite&.incr_tests_skipped_by_tia_count
|
|
242
243
|
end
|
|
243
244
|
|
|
244
245
|
# Handle code coverage
|
|
@@ -303,8 +304,8 @@ module Datadog
|
|
|
303
304
|
def mark_if_skippable(test)
|
|
304
305
|
return if !enabled? || !skipping_tests?
|
|
305
306
|
|
|
306
|
-
if skippable?(test.datadog_test_id) && !test.
|
|
307
|
-
test.
|
|
307
|
+
if skippable?(test.datadog_test_id) && !test.should_retry_attempt_to_fix?
|
|
308
|
+
test.set_internal_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
|
|
308
309
|
|
|
309
310
|
Datadog.logger.debug { "Marked test as skippable: #{test.datadog_test_id}" }
|
|
310
311
|
else
|
|
@@ -325,13 +326,13 @@ module Datadog
|
|
|
325
326
|
|
|
326
327
|
if unskippable
|
|
327
328
|
Utils::Telemetry.itr_forced_run
|
|
328
|
-
test_suite.
|
|
329
|
+
test_suite.set_internal_tag(Ext::Test::TAG_ITR_FORCED_RUN, "true")
|
|
329
330
|
|
|
330
331
|
Datadog.logger.debug { "Forced run of skippable test suite: #{test_suite.name}" }
|
|
331
332
|
return
|
|
332
333
|
end
|
|
333
334
|
|
|
334
|
-
test_suite.
|
|
335
|
+
test_suite.set_internal_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
|
|
335
336
|
test_suite.skipped!(reason: Ext::Test::SkipReason::TEST_IMPACT_ANALYSIS)
|
|
336
337
|
|
|
337
338
|
Datadog.logger.debug { "Marked test suite as skippable: #{test_suite.name}" }
|
|
@@ -349,7 +350,17 @@ module Datadog
|
|
|
349
350
|
end
|
|
350
351
|
|
|
351
352
|
def on_test_suite_finished(test_suite, context)
|
|
352
|
-
return unless enabled?
|
|
353
|
+
return unless enabled?
|
|
354
|
+
|
|
355
|
+
skipped_count = if suite_skipping_mode?
|
|
356
|
+
test_suite.skipped_by_test_impact_analysis? ? 1 : 0
|
|
357
|
+
else
|
|
358
|
+
test_suite.tests_skipped_by_tia_count
|
|
359
|
+
end
|
|
360
|
+
test_suite.set_internal_tag(Ext::Test::TAG_ITR_TESTS_SKIPPED, skipped_count.positive?.to_s)
|
|
361
|
+
test_suite.set_internal_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, skipped_count)
|
|
362
|
+
|
|
363
|
+
return unless suite_skipping_mode?
|
|
353
364
|
|
|
354
365
|
if test_suite.skipped_by_test_impact_analysis?
|
|
355
366
|
Telemetry.itr_skipped
|
|
@@ -379,8 +390,8 @@ module Datadog
|
|
|
379
390
|
Datadog.logger.debug { "Finished optimised session with test skipping enabled: #{@test_skipping_enabled}" }
|
|
380
391
|
Datadog.logger.debug { "#{skipped_tests_count} tests were skipped" }
|
|
381
392
|
|
|
382
|
-
test_session.
|
|
383
|
-
test_session.
|
|
393
|
+
test_session.set_internal_tag(Ext::Test::TAG_ITR_TESTS_SKIPPED, skipped_tests_count.positive?.to_s)
|
|
394
|
+
test_session.set_internal_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, skipped_tests_count)
|
|
384
395
|
end
|
|
385
396
|
|
|
386
397
|
def shutdown!
|
|
@@ -123,7 +123,7 @@ module Datadog
|
|
|
123
123
|
|
|
124
124
|
# mark the test session so that all events emitted in this session are tagged
|
|
125
125
|
# with the hidden _dd.ci.library_configuration_error.skippable_tests tag
|
|
126
|
-
test_session.
|
|
126
|
+
test_session.set_internal_tag(Ext::Test::LibraryConfigurationError::TAG_SKIPPABLE_TESTS, "true")
|
|
127
127
|
end
|
|
128
128
|
|
|
129
129
|
Response.from_http_response(http_response)
|
|
@@ -38,7 +38,7 @@ module Datadog
|
|
|
38
38
|
|
|
39
39
|
return unless @enabled
|
|
40
40
|
|
|
41
|
-
test_session.
|
|
41
|
+
test_session.set_internal_tag(Ext::Test::TAG_TEST_MANAGEMENT_ENABLED, "true")
|
|
42
42
|
|
|
43
43
|
return if load_component_state
|
|
44
44
|
|
|
@@ -64,9 +64,9 @@ module Datadog
|
|
|
64
64
|
|
|
65
65
|
Datadog.logger.debug { "Test properties for test #{datadog_test_id} are: [#{test_properties}]" }
|
|
66
66
|
|
|
67
|
-
test_span.
|
|
68
|
-
test_span.
|
|
69
|
-
test_span.
|
|
67
|
+
test_span.set_internal_tag(Ext::Test::TAG_IS_QUARANTINED, "true") if test_properties["quarantined"]
|
|
68
|
+
test_span.set_internal_tag(Ext::Test::TAG_IS_TEST_DISABLED, "true") if test_properties["disabled"]
|
|
69
|
+
test_span.set_internal_tag(Ext::Test::TAG_IS_ATTEMPT_TO_FIX, "true") if test_properties["attempt_to_fix"]
|
|
70
70
|
end
|
|
71
71
|
|
|
72
72
|
def attempt_to_fix?(datadog_fqn_test_id)
|
|
@@ -116,7 +116,7 @@ module Datadog
|
|
|
116
116
|
|
|
117
117
|
# mark the test session so that all events emitted in this session are tagged
|
|
118
118
|
# with the hidden _dd.ci.library_configuration_error.test_management_tests tag
|
|
119
|
-
test_session.
|
|
119
|
+
test_session.set_internal_tag(Ext::Test::LibraryConfigurationError::TAG_TEST_MANAGEMENT_TESTS, "true")
|
|
120
120
|
end
|
|
121
121
|
|
|
122
122
|
Response.from_http_response(http_response).tests
|
|
@@ -60,12 +60,15 @@ module Datadog
|
|
|
60
60
|
]
|
|
61
61
|
@mutex = Mutex.new
|
|
62
62
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
end
|
|
63
|
+
@dynamic_atr_enabled = dynamic_atr_enabled
|
|
64
|
+
@dynamic_atr_has_custom_buckets = !dynamic_atr_buckets.nil?
|
|
66
65
|
end
|
|
67
66
|
|
|
68
67
|
def configure(library_settings, test_session)
|
|
68
|
+
if @dynamic_atr_enabled
|
|
69
|
+
Utils::Telemetry.record_dynamic_atr_retries(has_custom_buckets: @dynamic_atr_has_custom_buckets)
|
|
70
|
+
end
|
|
71
|
+
|
|
69
72
|
# let all strategies configure themselves
|
|
70
73
|
@retry_strategies.each do |strategy|
|
|
71
74
|
strategy.configure(library_settings, test_session)
|
|
@@ -85,6 +88,8 @@ module Datadog
|
|
|
85
88
|
end
|
|
86
89
|
|
|
87
90
|
def build_driver(test_span)
|
|
91
|
+
return Driver::NoRetry.new if test_span.no_auto_retry?
|
|
92
|
+
|
|
88
93
|
@mutex.synchronize do
|
|
89
94
|
# find the first strategy that covers the test span and let it build the driver
|
|
90
95
|
strategy = @retry_strategies.find { |strategy| strategy.covers?(test_span) }
|
|
@@ -105,27 +110,19 @@ module Datadog
|
|
|
105
110
|
if current_retry_driver.nil?
|
|
106
111
|
# We always run test at least once and after the first pass create a correct retry driver
|
|
107
112
|
self.current_retry_driver = build_driver(test_span)
|
|
108
|
-
|
|
109
|
-
# When the driver decides no retries are needed after the first execution
|
|
110
|
-
# (e.g., attempt-to-fix test that failed immediately), we still need to tag
|
|
111
|
-
# the outcome for strategies that track retry results.
|
|
112
|
-
if !should_retry? && current_retry_driver&.tracks_retry_results?
|
|
113
|
-
tag_last_retry(test_span)
|
|
114
|
-
end
|
|
115
113
|
else
|
|
116
|
-
# After each retry we let the driver to record the result.
|
|
117
|
-
# Then the driver will decide if we should retry again.
|
|
118
114
|
current_retry_driver&.record_retry(test_span)
|
|
119
|
-
|
|
120
|
-
# We know that the test was already retried at least once so if we should not retry anymore, then this
|
|
121
|
-
# is the last retry.
|
|
122
|
-
tag_last_retry(test_span) unless should_retry?
|
|
123
115
|
end
|
|
124
116
|
|
|
125
117
|
# Some retry strategies such as Early Flake Detection change the number of retries based on
|
|
126
118
|
# how long the test was.
|
|
127
119
|
current_retry_driver&.record_duration(test_span.peek_duration)
|
|
128
120
|
|
|
121
|
+
if !should_retry? && current_retry_driver&.tracks_retry_results?
|
|
122
|
+
tag_last_retry(test_span)
|
|
123
|
+
end
|
|
124
|
+
record_failure_suppression(test_span)
|
|
125
|
+
|
|
129
126
|
# We need to set the final status of the test (what will be reported to the test framework) on the last execution
|
|
130
127
|
# no matter if test was retried or not
|
|
131
128
|
#
|
|
@@ -139,11 +136,13 @@ module Datadog
|
|
|
139
136
|
end
|
|
140
137
|
|
|
141
138
|
def tag_last_retry(test_span)
|
|
142
|
-
|
|
139
|
+
return if test_span&.no_auto_retry?
|
|
140
|
+
|
|
141
|
+
test_span&.set_internal_tag(Ext::Test::TAG_HAS_FAILED_ALL_RETRIES, "true") if test_span&.all_executions_failed?
|
|
143
142
|
|
|
144
143
|
# if we are attempting to fix the test and all retries passed, we indicate that the fix might have worked
|
|
145
144
|
# otherwise we send "false" to show that it didn't work
|
|
146
|
-
test_span&.
|
|
145
|
+
test_span&.set_internal_tag(Ext::Test::TAG_ATTEMPT_TO_FIX_PASSED, test_span&.all_executions_passed?.to_s) if test_span&.should_retry_attempt_to_fix?
|
|
147
146
|
end
|
|
148
147
|
|
|
149
148
|
def should_retry?
|
|
@@ -154,6 +153,23 @@ module Datadog
|
|
|
154
153
|
|
|
155
154
|
private
|
|
156
155
|
|
|
156
|
+
def record_failure_suppression(test_span)
|
|
157
|
+
return unless test_span.failed?
|
|
158
|
+
|
|
159
|
+
reason = if test_span.should_quarantine? && !test_span.should_retry_attempt_to_fix?
|
|
160
|
+
Ext::Test::FailureSuppressionReason::QUARANTINE
|
|
161
|
+
else
|
|
162
|
+
retry_reason = current_retry_driver&.retry_reason
|
|
163
|
+
if retry_reason == Ext::Test::RetryReason::RETRY_FAILED && should_retry?
|
|
164
|
+
retry_reason
|
|
165
|
+
elsif retry_reason == Ext::Test::RetryReason::RETRY_DETECT_FLAKY && (should_retry? || test_span.any_retry_passed?)
|
|
166
|
+
retry_reason
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
test_span.set_internal_tag(Ext::Test::TAG_FAILURE_SUPPRESSION_REASON, reason) if reason
|
|
171
|
+
end
|
|
172
|
+
|
|
157
173
|
def test_tracing
|
|
158
174
|
Datadog.send(:components).test_tracing
|
|
159
175
|
end
|
|
@@ -16,8 +16,8 @@ module Datadog
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def mark_as_retry(test_span)
|
|
19
|
-
test_span&.
|
|
20
|
-
test_span&.
|
|
19
|
+
test_span&.set_internal_tag(Ext::Test::TAG_IS_RETRY, "true")
|
|
20
|
+
test_span&.set_internal_tag(Ext::Test::TAG_RETRY_REASON, retry_reason)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
def record_retry(test_span)
|
|
@@ -8,14 +8,15 @@ module Datadog
|
|
|
8
8
|
module CI
|
|
9
9
|
module TestRetries
|
|
10
10
|
module Driver
|
|
11
|
-
#
|
|
11
|
+
# repeat new tests within the configured total execution budget (early flake detection)
|
|
12
12
|
# stop early once both a pass and a fail have been observed (flakiness confirmed)
|
|
13
13
|
class RetryFlakeDetection < Base
|
|
14
14
|
def initialize(test_span, max_attempts_thresholds:)
|
|
15
15
|
@max_attempts_thresholds = max_attempts_thresholds
|
|
16
16
|
@attempts = 0
|
|
17
|
-
|
|
18
|
-
@
|
|
17
|
+
@max_attempts = 0
|
|
18
|
+
@duration_recorded = false
|
|
19
|
+
@test_span = test_span
|
|
19
20
|
|
|
20
21
|
# track outcomes to stop early once flakiness is confirmed
|
|
21
22
|
@passed_once = !!test_span&.passed?
|
|
@@ -37,7 +38,14 @@ module Datadog
|
|
|
37
38
|
end
|
|
38
39
|
|
|
39
40
|
def record_duration(duration)
|
|
40
|
-
|
|
41
|
+
return if @duration_recorded
|
|
42
|
+
|
|
43
|
+
total_executions = @max_attempts_thresholds.max_attempts_for_duration(duration)
|
|
44
|
+
@max_attempts = [total_executions - 1, 0].max
|
|
45
|
+
@duration_recorded = true
|
|
46
|
+
if total_executions.zero?
|
|
47
|
+
@test_span.set_internal_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_SLOW)
|
|
48
|
+
end
|
|
41
49
|
|
|
42
50
|
Datadog.logger.debug { "Recorded test duration of [#{duration}], new Max Attempts value is [#{@max_attempts}]" }
|
|
43
51
|
end
|
|
@@ -40,7 +40,7 @@ module Datadog
|
|
|
40
40
|
return unless @enabled
|
|
41
41
|
|
|
42
42
|
# mark early flake detection enabled for test session
|
|
43
|
-
test_session.
|
|
43
|
+
test_session.set_internal_tag(Ext::Test::TAG_EARLY_FLAKE_ENABLED, "true")
|
|
44
44
|
|
|
45
45
|
set_max_attempts_thresholds(library_settings)
|
|
46
46
|
calculate_total_retries_limit(library_settings, test_session)
|
|
@@ -58,7 +58,7 @@ module Datadog
|
|
|
58
58
|
private
|
|
59
59
|
|
|
60
60
|
def mark_test_session_faulty(test_session)
|
|
61
|
-
test_session&.
|
|
61
|
+
test_session&.set_internal_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
def set_max_attempts_thresholds(library_settings)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require_relative "base"
|
|
4
4
|
|
|
5
5
|
require_relative "../driver/retry_flaky_fixed"
|
|
6
|
+
require_relative "../../ext/settings"
|
|
6
7
|
|
|
7
8
|
module Datadog
|
|
8
9
|
module CI
|
|
@@ -17,18 +18,20 @@ module Datadog
|
|
|
17
18
|
max_attempts:
|
|
18
19
|
)
|
|
19
20
|
@enabled = enabled
|
|
20
|
-
@
|
|
21
|
+
@configured_max_attempts = max_attempts
|
|
22
|
+
@max_attempts = max_attempts || Ext::Settings::DEFAULT_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES
|
|
21
23
|
end
|
|
22
24
|
|
|
23
25
|
def configure(library_settings, test_session)
|
|
24
26
|
@enabled &&= library_settings.test_management_enabled?
|
|
25
|
-
@max_attempts = library_settings.attempt_to_fix_retries_count ||
|
|
27
|
+
@max_attempts = @configured_max_attempts || library_settings.attempt_to_fix_retries_count ||
|
|
28
|
+
Ext::Settings::DEFAULT_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES
|
|
26
29
|
end
|
|
27
30
|
|
|
28
31
|
def covers?(test_span)
|
|
29
32
|
return false unless @enabled
|
|
30
33
|
|
|
31
|
-
!!test_span&.
|
|
34
|
+
!!test_span&.should_retry_attempt_to_fix?
|
|
32
35
|
end
|
|
33
36
|
|
|
34
37
|
def build_driver(test_span)
|
|
@@ -25,6 +25,7 @@ module Datadog
|
|
|
25
25
|
# tracks final status for each test (the status that is reported after all retries):
|
|
26
26
|
# { "MySuite.mytest.a:1" => "pass" }
|
|
27
27
|
@final_statuses_per_test = {}
|
|
28
|
+
@tests_skipped_by_tia_count = 0
|
|
28
29
|
end
|
|
29
30
|
|
|
30
31
|
# Adds files that Test Impact Analysis should consider capable of
|
|
@@ -97,6 +98,16 @@ module Datadog
|
|
|
97
98
|
end
|
|
98
99
|
end
|
|
99
100
|
|
|
101
|
+
# @internal
|
|
102
|
+
def tests_skipped_by_tia_count
|
|
103
|
+
synchronize { @tests_skipped_by_tia_count }
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# @internal
|
|
107
|
+
def incr_tests_skipped_by_tia_count
|
|
108
|
+
synchronize { @tests_skipped_by_tia_count += 1 }
|
|
109
|
+
end
|
|
110
|
+
|
|
100
111
|
# @internal
|
|
101
112
|
def record_test_result(test_id, datadog_test_status)
|
|
102
113
|
synchronize do
|
|
@@ -59,6 +59,7 @@ module Datadog
|
|
|
59
59
|
@codeowners = codeowners
|
|
60
60
|
@logical_test_session_name = logical_test_session_name
|
|
61
61
|
@trace_setup_teardown_enabled = trace_setup_teardown_enabled
|
|
62
|
+
@configuration_locked = false
|
|
62
63
|
|
|
63
64
|
# "Known tests" feature fetches a list of all tests known to Datadog for this repository
|
|
64
65
|
# and uses this list to determine if a test is new or not. New tests are marked with "test.is_new" tag.
|
|
@@ -83,6 +84,10 @@ module Datadog
|
|
|
83
84
|
@local_test_suites_mode = true
|
|
84
85
|
end
|
|
85
86
|
|
|
87
|
+
def shared_tags
|
|
88
|
+
@context.shared_tags
|
|
89
|
+
end
|
|
90
|
+
|
|
86
91
|
def configure(library_configuration, test_session)
|
|
87
92
|
return unless library_configuration.known_tests_enabled?
|
|
88
93
|
|
|
@@ -94,6 +99,8 @@ module Datadog
|
|
|
94
99
|
end
|
|
95
100
|
|
|
96
101
|
def start_test_session(service: nil, tags: {}, estimated_total_tests_count: 0, distributed: nil, local_test_suites_mode: true)
|
|
102
|
+
# Forks inherit this flag; fresh workers acquire it when joining the distributed session.
|
|
103
|
+
@configuration_locked = true
|
|
97
104
|
@local_test_suites_mode = local_test_suites_mode
|
|
98
105
|
|
|
99
106
|
start_drb_service
|
|
@@ -123,7 +130,10 @@ module Datadog
|
|
|
123
130
|
test_suite
|
|
124
131
|
end
|
|
125
132
|
|
|
126
|
-
|
|
133
|
+
# Exclusion keywords are for framework instrumentation only; the public
|
|
134
|
+
# Datadog::CI.trace_test and .start_test APIs do not expose them.
|
|
135
|
+
# @internal_api
|
|
136
|
+
def trace_test(test_name, test_suite_name, service: nil, tags: {}, prevent_failure_suppression: false, no_auto_retry: false, &block)
|
|
127
137
|
test_name = Utils::TestName.normalize(test_name)
|
|
128
138
|
test_suite_name = Utils::TestName.normalize(test_suite_name)
|
|
129
139
|
|
|
@@ -132,6 +142,7 @@ module Datadog
|
|
|
132
142
|
|
|
133
143
|
if block
|
|
134
144
|
@context.trace_test(test_name, test_suite, service: service, tags: tags) do |test|
|
|
145
|
+
test.apply_automation_exclusions(prevent_failure_suppression: prevent_failure_suppression, no_auto_retry: no_auto_retry)
|
|
135
146
|
on_test_started(test)
|
|
136
147
|
res = block.call(test)
|
|
137
148
|
on_test_finished(test)
|
|
@@ -139,6 +150,7 @@ module Datadog
|
|
|
139
150
|
end
|
|
140
151
|
else
|
|
141
152
|
test = @context.trace_test(test_name, test_suite, service: service, tags: tags)
|
|
153
|
+
test.apply_automation_exclusions(prevent_failure_suppression: prevent_failure_suppression, no_auto_retry: no_auto_retry)
|
|
142
154
|
on_test_started(test)
|
|
143
155
|
test
|
|
144
156
|
end
|
|
@@ -167,6 +179,10 @@ module Datadog
|
|
|
167
179
|
maybe_remote_context.any_tests_started?
|
|
168
180
|
end
|
|
169
181
|
|
|
182
|
+
def configuration_locked?
|
|
183
|
+
@configuration_locked
|
|
184
|
+
end
|
|
185
|
+
|
|
170
186
|
def active_test_session
|
|
171
187
|
maybe_remote_context.active_test_session
|
|
172
188
|
rescue => e
|
|
@@ -204,6 +220,10 @@ module Datadog
|
|
|
204
220
|
on_test_session_finished(test_session) if test_session
|
|
205
221
|
|
|
206
222
|
@context.deactivate_test_session
|
|
223
|
+
# Only the session owner can release the lock. Distributed workers retain their
|
|
224
|
+
# remote feature state for their lifetime, including after the parent's session ends.
|
|
225
|
+
@configuration_locked = false unless client_process?
|
|
226
|
+
nil
|
|
207
227
|
end
|
|
208
228
|
|
|
209
229
|
def deactivate_test_module
|
|
@@ -278,6 +298,7 @@ module Datadog
|
|
|
278
298
|
|
|
279
299
|
# sets logical test session name if none provided by the user
|
|
280
300
|
override_logical_test_session_name!(test_session) if logical_test_session_name.nil?
|
|
301
|
+
@context.finalize_shared_tags
|
|
281
302
|
|
|
282
303
|
# Signal Remote::Component to configure the library.
|
|
283
304
|
# Note that it will call this component back (unfortunate circular dependency).
|
|
@@ -360,7 +381,7 @@ module Datadog
|
|
|
360
381
|
def set_codeowners(span)
|
|
361
382
|
source = span.source_file
|
|
362
383
|
owners = @codeowners.list_owners(source) if source
|
|
363
|
-
span.
|
|
384
|
+
span.set_internal_tag(Ext::Test::TAG_CODEOWNERS, owners) unless owners.nil?
|
|
364
385
|
end
|
|
365
386
|
|
|
366
387
|
def validate_source_location(test)
|
|
@@ -380,7 +401,7 @@ module Datadog
|
|
|
380
401
|
"end line (#{end_line}) is before the start line (#{start_line}). " \
|
|
381
402
|
"Removing #{Ext::Test::TAG_SOURCE_END} tag."
|
|
382
403
|
end
|
|
383
|
-
test.
|
|
404
|
+
test.clear_internal_tag(Ext::Test::TAG_SOURCE_END)
|
|
384
405
|
end
|
|
385
406
|
end
|
|
386
407
|
|
|
@@ -398,8 +419,8 @@ module Datadog
|
|
|
398
419
|
"but it was not found. Fixing it by assigning test suite [#{test_suite.name}] to the test."
|
|
399
420
|
end
|
|
400
421
|
|
|
401
|
-
test.
|
|
402
|
-
test.
|
|
422
|
+
test.set_internal_tag(Ext::Test::TAG_TEST_SUITE_ID, test_suite.id.to_s)
|
|
423
|
+
test.set_internal_tag(Ext::Test::TAG_SUITE, test_suite.name)
|
|
403
424
|
end
|
|
404
425
|
|
|
405
426
|
def validate_test_suite_level_visibility_correctness(test)
|
|
@@ -458,7 +479,7 @@ module Datadog
|
|
|
458
479
|
@known_tests_enabled = false
|
|
459
480
|
|
|
460
481
|
# this adds unfortunate knowledge on EFD from Testvisibility, rethink this
|
|
461
|
-
test_session&.
|
|
482
|
+
test_session&.set_internal_tag(Ext::Test::TAG_EARLY_FLAKE_ABORT_REASON, Ext::Test::EARLY_FLAKE_FAULTY)
|
|
462
483
|
|
|
463
484
|
Datadog.logger.warn("Empty set of tests known to Datadog")
|
|
464
485
|
end
|
|
@@ -476,7 +497,7 @@ module Datadog
|
|
|
476
497
|
end
|
|
477
498
|
|
|
478
499
|
def mark_test_as_new(test_span)
|
|
479
|
-
test_span.
|
|
500
|
+
test_span.set_internal_tag(Ext::Test::TAG_IS_NEW, "true")
|
|
480
501
|
end
|
|
481
502
|
|
|
482
503
|
def test_impact_analysis
|
|
@@ -10,6 +10,7 @@ require_relative "telemetry"
|
|
|
10
10
|
|
|
11
11
|
require_relative "../ext/app_types"
|
|
12
12
|
require_relative "../ext/environment"
|
|
13
|
+
require_relative "../ext/metadata"
|
|
13
14
|
require_relative "../ext/test"
|
|
14
15
|
|
|
15
16
|
require_relative "../utils/configuration"
|
|
@@ -43,6 +44,14 @@ module Datadog
|
|
|
43
44
|
@any_tests_started = false
|
|
44
45
|
end
|
|
45
46
|
|
|
47
|
+
def shared_tags
|
|
48
|
+
@shared_tags || Span::EMPTY_TAGS
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def finalize_shared_tags
|
|
52
|
+
@shared_tags = build_shared_tags
|
|
53
|
+
end
|
|
54
|
+
|
|
46
55
|
def start_test_session(service: nil, tags: {})
|
|
47
56
|
@process_context.fetch_or_activate_test_session do
|
|
48
57
|
@mutex.synchronize { @any_tests_started = false }
|
|
@@ -239,17 +248,25 @@ module Datadog
|
|
|
239
248
|
end
|
|
240
249
|
|
|
241
250
|
# TAGGING
|
|
251
|
+
def build_shared_tags
|
|
252
|
+
environment_tags = @shared_tags || Ext::Environment.tags(ENV)
|
|
253
|
+
.slice(*Ext::Metadata::SHARED_ENVIRONMENT_TAGS)
|
|
254
|
+
.transform_values { |value| Core::Utils.utf8_encode(value).dup.freeze }
|
|
255
|
+
tags = environment_tags.merge(Ext::Test::LibraryCapabilities::CAPABILITY_VERSIONS)
|
|
256
|
+
tags[Ext::Test::TAG_USER_PROVIDED_TEST_SERVICE] = Utils::Configuration.service_name_provided_by_user?.to_s.freeze
|
|
257
|
+
session_name = @test_tracing_component.logical_test_session_name
|
|
258
|
+
tags[Ext::Test::TAG_TEST_SESSION_NAME] = session_name.dup.freeze unless session_name.nil?
|
|
259
|
+
tags.freeze
|
|
260
|
+
end
|
|
261
|
+
|
|
242
262
|
def set_initial_tags(ci_span, tags)
|
|
243
|
-
@
|
|
263
|
+
@shared_tags ||= build_shared_tags
|
|
244
264
|
|
|
245
265
|
ci_span.set_default_tags
|
|
246
266
|
ci_span.set_environment_runtime_tags
|
|
247
|
-
|
|
248
|
-
ci_span.
|
|
249
|
-
ci_span.
|
|
250
|
-
ci_span.set_tags(@environment_tags)
|
|
251
|
-
|
|
252
|
-
ci_span.set_metric(Ext::Test::METRIC_CPU_COUNT, Utils::TestRun.virtual_cpu_count)
|
|
267
|
+
ci_span.set_internal_tags(@runtime_tags_overrides) unless @runtime_tags_overrides.empty?
|
|
268
|
+
ci_span.set_internal_tags(tags.reject { |key, _| Ext::Metadata::SHARED_TAGS.include?(key.to_s) })
|
|
269
|
+
ci_span.set_internal_metric(Ext::Test::METRIC_CPU_COUNT, Utils::TestRun.virtual_cpu_count)
|
|
253
270
|
end
|
|
254
271
|
|
|
255
272
|
# PROPAGATING CONTEXT FROM TOP-LEVEL TO THE LOWER LEVELS
|
|
@@ -28,7 +28,7 @@ module Datadog
|
|
|
28
28
|
return
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
test_session.
|
|
31
|
+
test_session.set_internal_tag(Ext::Test::TAG_CODE_COVERAGE_LINES_PCT, result.covered_percent)
|
|
32
32
|
rescue => e
|
|
33
33
|
Datadog.logger.warn("Failed to extract SimpleCov code coverage: #{e.class}: #{e.message}")
|
|
34
34
|
end
|
|
@@ -112,7 +112,7 @@ module Datadog
|
|
|
112
112
|
unless response.ok?
|
|
113
113
|
# mark the test session so that all events emitted in this session are tagged
|
|
114
114
|
# with the hidden _dd.ci.library_configuration_error.known_tests tag
|
|
115
|
-
test_session.
|
|
115
|
+
test_session.set_internal_tag(Ext::Test::LibraryConfigurationError::TAG_KNOWN_TESTS, "true")
|
|
116
116
|
|
|
117
117
|
Datadog.logger.debug(
|
|
118
118
|
"Failed to fetch known tests page ##{page_number}, bailing out of known tests fetch. " \
|
|
@@ -16,6 +16,10 @@ module Datadog
|
|
|
16
16
|
@local_test_suites_mode = true
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
def shared_tags
|
|
20
|
+
{}
|
|
21
|
+
end
|
|
22
|
+
|
|
19
23
|
def configure(_, _)
|
|
20
24
|
end
|
|
21
25
|
|
|
@@ -33,7 +37,8 @@ module Datadog
|
|
|
33
37
|
skip_tracing
|
|
34
38
|
end
|
|
35
39
|
|
|
36
|
-
|
|
40
|
+
# @internal_api
|
|
41
|
+
def trace_test(test_name, test_suite_name, service: nil, tags: {}, prevent_failure_suppression: false, no_auto_retry: false, &block)
|
|
37
42
|
skip_tracing(block)
|
|
38
43
|
end
|
|
39
44
|
|
|
@@ -51,6 +56,10 @@ module Datadog
|
|
|
51
56
|
false
|
|
52
57
|
end
|
|
53
58
|
|
|
59
|
+
def configuration_locked?
|
|
60
|
+
false
|
|
61
|
+
end
|
|
62
|
+
|
|
54
63
|
def active_test_session
|
|
55
64
|
end
|
|
56
65
|
|
|
@@ -7,12 +7,10 @@ require "datadog/core/utils/only_once"
|
|
|
7
7
|
require_relative "serializers/factories/test_suite_level"
|
|
8
8
|
require_relative "serializers/meta_truncation"
|
|
9
9
|
|
|
10
|
-
require_relative "../ext/app_types"
|
|
11
10
|
require_relative "../ext/telemetry"
|
|
12
11
|
require_relative "../ext/transport"
|
|
13
12
|
require_relative "../transport/event_platform_transport"
|
|
14
13
|
require_relative "../transport/telemetry"
|
|
15
|
-
require_relative "../utils/configuration"
|
|
16
14
|
|
|
17
15
|
module Datadog
|
|
18
16
|
module CI
|
|
@@ -96,7 +94,7 @@ module Datadog
|
|
|
96
94
|
packer.write(1)
|
|
97
95
|
|
|
98
96
|
packer.write("metadata")
|
|
99
|
-
packer.write_map_header(
|
|
97
|
+
packer.write_map_header(2)
|
|
100
98
|
|
|
101
99
|
packer.write("*")
|
|
102
100
|
metadata_fields_count = dd_env ? 4 : 3
|
|
@@ -116,25 +114,8 @@ module Datadog
|
|
|
116
114
|
packer.write("library_version")
|
|
117
115
|
packer.write(Serializers::MetaTruncation.truncate_value(Datadog::CI::VERSION::STRING))
|
|
118
116
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
Ext::AppTypes::CI_SPAN_TYPES.each do |ci_span_type|
|
|
122
|
-
packer.write(ci_span_type)
|
|
123
|
-
packer.write_map_header(2 + library_capabilities_tags.count)
|
|
124
|
-
|
|
125
|
-
packer.write(Ext::Test::TAG_TEST_SESSION_NAME)
|
|
126
|
-
packer.write(Serializers::MetaTruncation.truncate_value(test_tracing&.logical_test_session_name))
|
|
127
|
-
|
|
128
|
-
packer.write(Ext::Test::TAG_USER_PROVIDED_TEST_SERVICE)
|
|
129
|
-
packer.write(
|
|
130
|
-
Serializers::MetaTruncation.truncate_value(Utils::Configuration.service_name_provided_by_user?.to_s)
|
|
131
|
-
)
|
|
132
|
-
|
|
133
|
-
library_capabilities_tags.each do |tag, value|
|
|
134
|
-
packer.write(tag)
|
|
135
|
-
packer.write(Serializers::MetaTruncation.truncate_value(value))
|
|
136
|
-
end
|
|
137
|
-
end
|
|
117
|
+
packer.write("test_levels")
|
|
118
|
+
packer.write(Serializers::MetaTruncation.truncate_string_values(test_tracing&.shared_tags || {}))
|
|
138
119
|
|
|
139
120
|
packer.write("events")
|
|
140
121
|
end
|