dd-trace 5.116.0 → 5.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ci/diagnose.js +162 -21
- package/ci/init.js +0 -1
- package/ci/runbook.md +187 -180
- package/ci/test-optimization-validation/approval-artifacts.js +3 -1
- package/ci/test-optimization-validation/approval.js +52 -15
- package/ci/test-optimization-validation/ci-command-candidate.js +26 -32
- package/ci/test-optimization-validation/ci-discovery.js +1 -1
- package/ci/test-optimization-validation/ci-package-scripts.js +141 -0
- package/ci/test-optimization-validation/ci-remediation.js +112 -39
- package/ci/test-optimization-validation/cli.js +516 -681
- package/ci/test-optimization-validation/command-blocker.js +188 -33
- package/ci/test-optimization-validation/command-output-policy.js +3 -26
- package/ci/test-optimization-validation/command-runner.js +189 -245
- package/ci/test-optimization-validation/environment.js +90 -0
- package/ci/test-optimization-validation/executable.js +159 -388
- package/ci/test-optimization-validation/framework-adapters/cucumber.js +119 -0
- package/ci/test-optimization-validation/framework-adapters/cypress.js +107 -0
- package/ci/test-optimization-validation/framework-adapters/playwright.js +181 -0
- package/ci/test-optimization-validation/generated-files.js +75 -13
- package/ci/test-optimization-validation/generated-test-contract.js +283 -0
- package/ci/test-optimization-validation/generated-verifier.js +49 -16
- package/ci/test-optimization-validation/literal-environment.js +57 -0
- package/ci/test-optimization-validation/manifest-loader.js +2 -22
- package/ci/test-optimization-validation/manifest-scaffold.js +816 -515
- package/ci/test-optimization-validation/manifest-schema.js +494 -702
- package/ci/test-optimization-validation/offline-fixtures.js +4 -1
- package/ci/test-optimization-validation/plan-writer.js +243 -984
- package/ci/test-optimization-validation/preflight-runner.js +49 -55
- package/ci/test-optimization-validation/redaction.js +2 -1
- package/ci/test-optimization-validation/report-writer.js +391 -1357
- package/ci/test-optimization-validation/result-semantics.js +86 -0
- package/ci/test-optimization-validation/runner-command.js +249 -0
- package/ci/test-optimization-validation/runner-contract.js +664 -0
- package/ci/test-optimization-validation/scenarios/auto-test-retries.js +29 -3
- package/ci/test-optimization-validation/scenarios/basic-reporting.js +240 -543
- package/ci/test-optimization-validation/scenarios/ci-wiring.js +450 -670
- package/ci/test-optimization-validation/scenarios/early-flake-detection.js +4 -3
- package/ci/test-optimization-validation/scenarios/helpers.js +67 -9
- package/ci/test-optimization-validation/scenarios/test-management.js +4 -3
- package/ci/test-optimization-validation/source-text.js +87 -0
- package/ci/test-optimization-validation/test-output.js +12 -22
- package/ext/tags.d.ts +1 -0
- package/index.d.ts +7 -0
- package/package.json +3 -3
- package/packages/datadog-instrumentations/src/cucumber.js +11 -12
- package/packages/datadog-instrumentations/src/helpers/channel.js +74 -0
- package/packages/datadog-instrumentations/src/helpers/hooks.js +2 -0
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js +1 -0
- package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/openai-agents.js +31 -0
- package/packages/datadog-instrumentations/src/jest.js +18 -17
- package/packages/datadog-instrumentations/src/mocha/main.js +79 -30
- package/packages/datadog-instrumentations/src/openai-agents.js +159 -0
- package/packages/datadog-instrumentations/src/playwright.js +3 -24
- package/packages/datadog-instrumentations/src/vitest-main-no-worker-init.js +1 -1
- package/packages/datadog-instrumentations/src/vitest-main.js +14 -26
- package/packages/datadog-instrumentations/src/vitest-util.js +0 -7
- package/packages/datadog-instrumentations/src/vitest-worker.js +5 -8
- package/packages/datadog-plugin-cucumber/src/index.js +2 -1
- package/packages/datadog-plugin-mocha/src/index.js +2 -1
- package/packages/datadog-plugin-openai-agents/src/index.js +74 -0
- package/packages/datadog-plugin-openai-agents/src/integration.js +503 -0
- package/packages/datadog-plugin-openai-agents/src/processor.js +108 -0
- package/packages/datadog-plugin-openai-agents/src/util.js +60 -0
- package/packages/datadog-plugin-playwright/src/index.js +4 -1
- package/packages/datadog-plugin-vitest/src/index.js +5 -5
- package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +48 -49
- package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +161 -33
- package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache-schema.js +6 -4
- package/packages/dd-trace/src/config/generated-config-types.d.ts +2 -0
- package/packages/dd-trace/src/config/supported-configurations.json +7 -0
- package/packages/dd-trace/src/encode/span-stats.js +7 -1
- package/packages/dd-trace/src/exporter.js +16 -9
- package/packages/dd-trace/src/llmobs/index.js +9 -1
- package/packages/dd-trace/src/llmobs/plugins/ai/util.js +104 -22
- package/packages/dd-trace/src/llmobs/plugins/openai/index.js +5 -9
- package/packages/dd-trace/src/llmobs/plugins/openai/utils.js +20 -0
- package/packages/dd-trace/src/llmobs/plugins/openai-agents/utils.js +321 -0
- package/packages/dd-trace/src/llmobs/sdk.js +2 -1
- package/packages/dd-trace/src/llmobs/span_processor.js +7 -2
- package/packages/dd-trace/src/llmobs/tagger.js +23 -3
- package/packages/dd-trace/src/llmobs/util.js +56 -3
- package/packages/dd-trace/src/openfeature/agentless_configuration_source.js +3 -6
- package/packages/dd-trace/src/openfeature/configuration_source.js +5 -8
- package/packages/dd-trace/src/opentelemetry/metrics/otlp_span_stats_transformer.js +5 -1
- package/packages/dd-trace/src/plugins/ci_plugin.js +9 -3
- package/packages/dd-trace/src/plugins/index.js +1 -0
- package/packages/dd-trace/src/plugins/outbound.js +4 -1
- package/packages/dd-trace/src/profiling/profilers/space.js +1 -1
- package/packages/dd-trace/src/profiling/profilers/wall.js +1 -0
- package/packages/dd-trace/src/span_processor.js +1 -1
- package/packages/dd-trace/src/span_stats.js +22 -4
- package/ci/test-optimization-validation/command-suitability.js +0 -471
- package/ci/test-optimization-validation/init-probe-preload.js +0 -163
- package/ci/test-optimization-validation/init-probe.js +0 -246
- package/ci/test-optimization-validation/late-initialization.js +0 -63
- package/ci/test-optimization-validation/local-command.js +0 -163
- package/ci/test-optimization-validation/setup-runner.js +0 -97
- package/ci/test-optimization-validation-manifest.schema.json +0 -1
|
@@ -2,1507 +2,541 @@
|
|
|
2
2
|
|
|
3
3
|
/* eslint-disable no-console */
|
|
4
4
|
|
|
5
|
-
const
|
|
6
|
-
const path = require('path')
|
|
5
|
+
const path = require('node:path')
|
|
7
6
|
|
|
8
|
-
const { buildCiCommandCandidate } = require('./ci-command-candidate')
|
|
9
7
|
const { sanitizeConsoleText, sanitizeForReport, sanitizeString } = require('./redaction')
|
|
10
8
|
const { writeFileSafely } = require('./safe-files')
|
|
11
9
|
|
|
12
|
-
const
|
|
10
|
+
const REPORT_FILENAME = 'report.md'
|
|
13
11
|
const SHARING_WARNING =
|
|
14
|
-
'
|
|
15
|
-
'
|
|
16
|
-
|
|
17
|
-
'
|
|
18
|
-
|
|
19
|
-
'
|
|
20
|
-
'
|
|
12
|
+
'This local diagnostic may contain repository paths, package names, CI metadata, commands, and sanitized output. ' +
|
|
13
|
+
'Review it before sharing outside trusted support or engineering channels.'
|
|
14
|
+
const SCENARIO_NAMES = {
|
|
15
|
+
all: 'Setup',
|
|
16
|
+
atr: 'Auto Test Retries',
|
|
17
|
+
'basic-reporting': 'Basic Reporting',
|
|
18
|
+
'ci-wiring': 'CI configuration',
|
|
19
|
+
efd: 'Early Flake Detection',
|
|
20
|
+
'generated-test-verification': 'Temporary test verification',
|
|
21
|
+
'test-management': 'Test Management',
|
|
22
|
+
}
|
|
21
23
|
|
|
24
|
+
/**
|
|
25
|
+
* Writes the final human-readable report and compact console summary.
|
|
26
|
+
*
|
|
27
|
+
* @param {object} input report inputs
|
|
28
|
+
* @param {object} input.manifest normalized manifest
|
|
29
|
+
* @param {object[]} input.results validation results
|
|
30
|
+
* @param {string} input.out output directory
|
|
31
|
+
* @param {object} [input.staticDiagnosis] static diagnosis artifacts
|
|
32
|
+
* @param {object} [input.runSummary] run summary
|
|
33
|
+
* @returns {void}
|
|
34
|
+
*/
|
|
22
35
|
function writeReport ({ manifest, results, out, staticDiagnosis, runSummary = {} }) {
|
|
23
|
-
const reportPath = path.join(out,
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
report: reportPath,
|
|
27
|
-
reportPath,
|
|
28
|
-
staticDiagnosis: staticDiagnosis && staticDiagnosis.reportPath,
|
|
29
|
-
}
|
|
30
|
-
const frameworkLabels = getFrameworkLabels(manifest)
|
|
31
|
-
const sanitizedResults = addNotSelectedResults({
|
|
32
|
-
manifest,
|
|
33
|
-
results: sanitizeForReport(results),
|
|
34
|
-
runSummary,
|
|
35
|
-
}).map(result => ({
|
|
36
|
+
const reportPath = path.join(out, REPORT_FILENAME)
|
|
37
|
+
const labels = getFrameworkLabels(manifest)
|
|
38
|
+
const sanitizedResults = sanitizeForReport(results).map(result => ({
|
|
36
39
|
...result,
|
|
37
|
-
frameworkDisplayName:
|
|
40
|
+
frameworkDisplayName: labels.get(result.frameworkId) || result.frameworkId,
|
|
38
41
|
}))
|
|
39
|
-
const report = {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
manifestPath: manifest.__path,
|
|
44
|
-
ciDiscovery: sanitizeForReport(manifest.ciDiscovery),
|
|
45
|
-
ciCommandCandidates: sanitizeForReport(getCiCommandCandidates(manifest, frameworkLabels)),
|
|
46
|
-
omitted: sanitizeForReport(getStringArray(manifest.omitted)),
|
|
47
|
-
omittedTestCommands: sanitizeForReport(
|
|
48
|
-
Array.isArray(manifest.omittedTestCommands) ? manifest.omittedTestCommands : []
|
|
49
|
-
),
|
|
42
|
+
const report = renderReport({
|
|
43
|
+
manifest,
|
|
44
|
+
out,
|
|
45
|
+
reportPath,
|
|
50
46
|
results: sanitizedResults,
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
validationSummaries: buildDiagnosticValidationSummaries({
|
|
57
|
-
manifest,
|
|
58
|
-
results: sanitizedResults,
|
|
59
|
-
reportDirectory: out,
|
|
60
|
-
}),
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
writeFileSafely(out, reportPath, renderMarkdown(report), 'Markdown report')
|
|
64
|
-
|
|
65
|
-
console.log(sanitizeConsoleText(renderConsoleSummary(sanitizedResults, reportPath, report.runSummary)))
|
|
47
|
+
runSummary: sanitizeForReport(runSummary),
|
|
48
|
+
staticDiagnosisPath: staticDiagnosis?.reportPath,
|
|
49
|
+
})
|
|
50
|
+
writeFileSafely(out, reportPath, report, 'Markdown report')
|
|
51
|
+
console.log(sanitizeConsoleText(renderConsole(sanitizedResults, runSummary, reportPath)))
|
|
66
52
|
}
|
|
67
53
|
|
|
68
54
|
/**
|
|
69
|
-
* Writes an explicit
|
|
55
|
+
* Writes an explicit pending report before project code runs.
|
|
70
56
|
*
|
|
71
57
|
* @param {object} input pending report inputs
|
|
72
58
|
* @param {object} input.manifest normalized manifest
|
|
73
|
-
* @param {string} input.out
|
|
59
|
+
* @param {string} input.out output directory
|
|
74
60
|
* @returns {void}
|
|
75
61
|
*/
|
|
76
62
|
function writePendingReport ({ manifest, out }) {
|
|
77
|
-
const reportPath = path.join(out,
|
|
78
|
-
const runSummary = { runCompleted: false, validatorExitCode: null }
|
|
79
|
-
const diagnosticJson = JSON.stringify({
|
|
80
|
-
version: 2,
|
|
81
|
-
runSummary,
|
|
82
|
-
validationSummaries: [],
|
|
83
|
-
artifacts: {
|
|
84
|
-
markdownReport: 'report.md',
|
|
85
|
-
manifest: relativeArtifactPath(manifest.__path, out),
|
|
86
|
-
},
|
|
87
|
-
}, null, 2)
|
|
63
|
+
const reportPath = path.join(out, REPORT_FILENAME)
|
|
88
64
|
writeFileSafely(out, reportPath, [
|
|
89
65
|
'# Datadog Test Optimization Validation Report',
|
|
90
66
|
'',
|
|
91
|
-
'
|
|
92
|
-
'Validator exit code: not available because the validation has not completed',
|
|
93
|
-
'',
|
|
94
|
-
'> Validation started but has not completed. Rerun the already-approved validator command before drawing a ' +
|
|
95
|
-
'Test Optimization conclusion.',
|
|
96
|
-
'',
|
|
97
|
-
`> ${SHARING_WARNING}`,
|
|
67
|
+
'**Status: INCOMPLETE**',
|
|
98
68
|
'',
|
|
99
|
-
'
|
|
69
|
+
'Validation started but did not finish. Do not draw a Test Optimization conclusion from this file.',
|
|
100
70
|
'',
|
|
101
|
-
|
|
102
|
-
diagnosticJson,
|
|
103
|
-
'```',
|
|
71
|
+
`Manifest: ${code(relative(out, manifest.__path))}`,
|
|
104
72
|
'',
|
|
105
|
-
|
|
106
|
-
'',
|
|
107
|
-
`Manifest: ${manifest.__path}`,
|
|
73
|
+
`> ${SHARING_WARNING}`,
|
|
108
74
|
'',
|
|
109
|
-
].join('\n'), '
|
|
75
|
+
].join('\n'), 'pending Markdown report')
|
|
110
76
|
}
|
|
111
77
|
|
|
112
|
-
|
|
78
|
+
/**
|
|
79
|
+
* Renders the final Markdown report.
|
|
80
|
+
*
|
|
81
|
+
* @param {object} input rendering inputs
|
|
82
|
+
* @param {object} input.manifest normalized manifest
|
|
83
|
+
* @param {string} input.out output directory
|
|
84
|
+
* @param {string} input.reportPath report path
|
|
85
|
+
* @param {object[]} input.results sanitized results
|
|
86
|
+
* @param {object} input.runSummary run summary
|
|
87
|
+
* @param {string|undefined} input.staticDiagnosisPath static diagnosis path
|
|
88
|
+
* @returns {string} Markdown
|
|
89
|
+
*/
|
|
90
|
+
function renderReport ({ manifest, out, reportPath, results, runSummary, staticDiagnosisPath }) {
|
|
113
91
|
const lines = [
|
|
114
92
|
'# Datadog Test Optimization Validation Report',
|
|
115
93
|
'',
|
|
116
|
-
|
|
117
|
-
`Validation completed: ${report.runSummary.runCompleted === true ? 'yes' : 'no'}`,
|
|
118
|
-
`Validator exit code: ${report.runSummary.validatorExitCode ?? 'not recorded'}`,
|
|
119
|
-
`Validation coverage: ${report.runSummary.validationCoverage || 'not recorded'}`,
|
|
94
|
+
`**Status: ${formatExecutionStatus(runSummary.executionStatus)}**`,
|
|
120
95
|
'',
|
|
121
|
-
|
|
96
|
+
`Coverage: ${runSummary.validationCoverage === 'complete' ? 'complete' : 'partial'}`,
|
|
97
|
+
`Validator exit code: ${runSummary.validatorExitCode ?? 'not recorded'}`,
|
|
98
|
+
`Cleanup: ${formatCleanupStatus(runSummary.cleanup)}`,
|
|
122
99
|
'',
|
|
123
|
-
`> ${
|
|
100
|
+
`> ${SHARING_WARNING}`,
|
|
101
|
+
'',
|
|
102
|
+
'## What This Means',
|
|
124
103
|
'',
|
|
125
|
-
|
|
104
|
+
...getVerdicts(results),
|
|
126
105
|
'',
|
|
127
|
-
'##
|
|
106
|
+
'## Results',
|
|
128
107
|
'',
|
|
108
|
+
'| Project / framework | Check | Result | Meaning |',
|
|
109
|
+
'| --- | --- | --- | --- |',
|
|
129
110
|
]
|
|
130
111
|
|
|
131
|
-
for (const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
appendMarkdownCiDiscovery(lines, report.ciDiscovery)
|
|
137
|
-
appendMarkdownStaticDiagnosisNotes(lines, report.staticDiagnosisNotes)
|
|
138
|
-
appendMarkdownCiCommandCandidates(lines, report.ciCommandCandidates)
|
|
139
|
-
appendMarkdownResultDetails(lines, report.results, path.dirname(report.artifacts.report))
|
|
140
|
-
|
|
141
|
-
lines.push('', '## Key Artifacts', '')
|
|
142
|
-
for (const [name, artifactPath] of getKeyArtifacts(report.artifacts)) {
|
|
143
|
-
if (!artifactPath) continue
|
|
144
|
-
lines.push(`- ${name}: ${markdownCode(artifactPath)}`)
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
relativizeHumanLines(lines, report.repositoryRoot)
|
|
148
|
-
appendMarkdownJsonSection(lines, 'Diagnostic JSON', buildCompactDiagnosticSummary(report))
|
|
149
|
-
|
|
150
|
-
return lines.join('\n')
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
function buildCompactDiagnosticSummary (report) {
|
|
154
|
-
const reportDirectory = path.dirname(report.artifacts.report)
|
|
155
|
-
|
|
156
|
-
return {
|
|
157
|
-
version: 2,
|
|
158
|
-
runSummary: report.runSummary,
|
|
159
|
-
validationSummaries: report.validationSummaries,
|
|
160
|
-
artifacts: compactArtifacts(report.artifacts, reportDirectory),
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
function buildDiagnosticValidationSummaries ({ manifest, results, reportDirectory }) {
|
|
165
|
-
const frameworks = new Map((manifest.frameworks || []).map(framework => [framework.id, framework]))
|
|
166
|
-
const grouped = new Map()
|
|
167
|
-
for (const result of results) {
|
|
168
|
-
const values = grouped.get(result.frameworkId) || []
|
|
169
|
-
values.push(result)
|
|
170
|
-
grouped.set(result.frameworkId, values)
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
const summaries = []
|
|
174
|
-
for (const [frameworkId, frameworkResults] of grouped) {
|
|
175
|
-
const framework = frameworks.get(frameworkId)
|
|
176
|
-
const checks = frameworkResults.map(result => buildDiagnosticCheck(result, reportDirectory)).filter(Boolean)
|
|
177
|
-
summaries.push(sanitizeForReport({
|
|
178
|
-
frameworkId,
|
|
179
|
-
status: getDiagnosticStatus(checks),
|
|
180
|
-
framework: buildDiagnosticFramework(framework, frameworkId),
|
|
181
|
-
ciCommandCandidate: compactCiCommandCandidate(
|
|
182
|
-
buildCiCommandCandidate(framework || {}),
|
|
183
|
-
manifest.repository?.root
|
|
184
|
-
),
|
|
185
|
-
checks,
|
|
186
|
-
}))
|
|
112
|
+
for (const result of getVisibleResults(results)) {
|
|
113
|
+
lines.push(
|
|
114
|
+
`| ${cell(result.frameworkDisplayName)} | ${cell(getScenarioName(result.scenario))} | ` +
|
|
115
|
+
`${cell(getDisplayStatus(result))} | ${cell(result.diagnosis)} |`
|
|
116
|
+
)
|
|
187
117
|
}
|
|
188
|
-
return summaries
|
|
189
|
-
}
|
|
190
118
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
if (
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const incomplete = isIncompleteResult(result)
|
|
200
|
-
const remediation = !incomplete && ['fail', 'error', 'blocked'].includes(result.status)
|
|
201
|
-
? getResultRecommendations(result)
|
|
202
|
-
: []
|
|
203
|
-
return {
|
|
204
|
-
id: definition.id,
|
|
205
|
-
name: definition.name,
|
|
206
|
-
status: incomplete ? 'unknown' : toDiagnosticStatus(result.status),
|
|
207
|
-
reason: staticOnly || ['fail', 'error', 'blocked'].includes(result.status) ? result.diagnosis : undefined,
|
|
208
|
-
command: staticOnly || checkLevelFailure || ciReplayUnavailable ? undefined : command?.command,
|
|
209
|
-
exitCode: staticOnly || checkLevelFailure || ciReplayUnavailable || result.evidence?.commandExitCode === undefined
|
|
210
|
-
? undefined
|
|
211
|
-
: String(result.evidence.commandExitCode),
|
|
212
|
-
evidence: staticOnly || checkLevelFailure
|
|
213
|
-
? undefined
|
|
214
|
-
: compactResultEvidence(definition.id, result.evidence || {}),
|
|
215
|
-
remediation: remediation.length > 0 ? remediation : undefined,
|
|
216
|
-
artifactDirectory: getRelativeArtifactDirectory(result.artifacts, reportDirectory),
|
|
119
|
+
const actions = getActions(results)
|
|
120
|
+
lines.push('', '## Next Actions', '')
|
|
121
|
+
if (actions.length === 0) {
|
|
122
|
+
lines.push('No corrective action was identified by the completed checks.')
|
|
123
|
+
} else {
|
|
124
|
+
for (const action of actions) {
|
|
125
|
+
lines.push(`- **${plain(action.framework)} / ${plain(action.check)}:** ${plain(action.text)}`)
|
|
126
|
+
}
|
|
217
127
|
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
function isDiagnosticCheckLevelFailure (result) {
|
|
221
|
-
if (!['fail', 'error'].includes(result.status)) return false
|
|
222
|
-
if (result.evidence?.commandExitCode !== undefined) return false
|
|
223
|
-
return !readResultCommand(result)
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
function getDiagnosticCheckDefinition (scenario) {
|
|
227
|
-
return {
|
|
228
|
-
all: { id: 'basic-reporting', name: 'Basic reporting' },
|
|
229
|
-
'basic-reporting': { id: 'basic-reporting', name: 'Basic reporting' },
|
|
230
|
-
'ci-wiring': { id: 'ci-wiring', name: 'CI wiring' },
|
|
231
|
-
'generated-test-verification': {
|
|
232
|
-
id: 'generated-test-verification',
|
|
233
|
-
name: 'Can the temporary validation test run?',
|
|
234
|
-
},
|
|
235
|
-
efd: { id: 'efd-new-test-detection-and-retry', name: 'EFD new test detection and retry' },
|
|
236
|
-
atr: { id: 'auto-test-retries', name: 'Auto test retries' },
|
|
237
|
-
'test-management': { id: 'test-management', name: 'Test Management' },
|
|
238
|
-
}[scenario]
|
|
239
|
-
}
|
|
240
128
|
|
|
241
|
-
|
|
242
|
-
if (
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
129
|
+
const diagnosticResults = results.filter(result => shouldRenderDiagnostics(result))
|
|
130
|
+
if (diagnosticResults.length > 0) {
|
|
131
|
+
lines.push('', '## Debugging Evidence', '')
|
|
132
|
+
for (const result of diagnosticResults) {
|
|
133
|
+
lines.push(
|
|
134
|
+
`### ${plain(result.frameworkDisplayName)}: ${plain(getScenarioName(result.scenario))}`,
|
|
135
|
+
'',
|
|
136
|
+
plain(result.diagnosis),
|
|
137
|
+
''
|
|
138
|
+
)
|
|
139
|
+
const artifactDirectory = getArtifactDirectory(result.artifacts, out)
|
|
140
|
+
if (artifactDirectory) {
|
|
141
|
+
lines.push(
|
|
142
|
+
`Artifacts: ${code(artifactDirectory)}`,
|
|
143
|
+
'',
|
|
144
|
+
'The directory may contain `command.json`, `stdout.txt`, `stderr.txt`, `events.ndjson`, and `result.json`.',
|
|
145
|
+
''
|
|
146
|
+
)
|
|
147
|
+
}
|
|
148
|
+
const evidence = compactEvidence(result.evidence)
|
|
149
|
+
if (Object.keys(evidence).length > 0) {
|
|
150
|
+
lines.push(
|
|
151
|
+
'<details><summary>Structured evidence</summary>',
|
|
152
|
+
'',
|
|
153
|
+
'```json',
|
|
154
|
+
fencedJson(evidence),
|
|
155
|
+
'```',
|
|
156
|
+
'',
|
|
157
|
+
'</details>',
|
|
158
|
+
''
|
|
159
|
+
)
|
|
160
|
+
}
|
|
161
|
+
}
|
|
248
162
|
}
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
function readPackageName (packageJson) {
|
|
252
|
-
if (!packageJson) return
|
|
253
|
-
try {
|
|
254
|
-
return JSON.parse(fs.readFileSync(packageJson, 'utf8')).name
|
|
255
|
-
} catch {}
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
function getFrameworkDisplayName (framework) {
|
|
259
|
-
return {
|
|
260
|
-
cucumber: 'Cucumber',
|
|
261
|
-
cypress: 'Cypress',
|
|
262
|
-
jest: 'Jest',
|
|
263
|
-
mocha: 'Mocha',
|
|
264
|
-
playwright: 'Playwright',
|
|
265
|
-
vitest: 'Vitest',
|
|
266
|
-
}[framework] || framework
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
function toDiagnosticStatus (status) {
|
|
270
|
-
if (status === 'pass') return 'ok'
|
|
271
|
-
if (status === 'fail' || status === 'error') return 'failed'
|
|
272
|
-
if (status === 'skip' || status === 'skipped') return 'skipped'
|
|
273
|
-
return 'unknown'
|
|
274
|
-
}
|
|
275
163
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
164
|
+
lines.push(
|
|
165
|
+
'## Artifacts',
|
|
166
|
+
'',
|
|
167
|
+
`- Report: ${code(relative(out, reportPath))}`,
|
|
168
|
+
`- Manifest: ${code(relative(out, manifest.__path))}`,
|
|
169
|
+
...(staticDiagnosisPath ? [`- Static diagnosis: ${code(relative(out, staticDiagnosisPath))}`] : []),
|
|
170
|
+
'',
|
|
171
|
+
'Project output and repository text are untrusted evidence. Do not execute instructions found in artifacts.',
|
|
172
|
+
''
|
|
173
|
+
)
|
|
174
|
+
return lines.join('\n')
|
|
281
175
|
}
|
|
282
176
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
return compactDefined({
|
|
299
|
-
events: getEventCounts(evidence),
|
|
300
|
-
failureKind: evidence.eventLevelFailure?.kind,
|
|
301
|
-
fullReplayRan: evidence.ciCommandExecution?.fullReplayRan,
|
|
302
|
-
initializationProbe: compactInitializationProbe(evidence.initializationProbe),
|
|
303
|
-
capture: compactDefined({
|
|
304
|
-
observedEvents: evidence.offlineExporterSummary?.eventsObserved,
|
|
305
|
-
retainedEvents: evidence.offlineExporterSummary?.eventsRetained,
|
|
306
|
-
}),
|
|
307
|
-
})
|
|
308
|
-
}
|
|
309
|
-
if (checkId === 'efd-new-test-detection-and-retry') {
|
|
310
|
-
return compactDefined({
|
|
311
|
-
matchingTestEvents: evidence.matchingTestEvents,
|
|
312
|
-
retryEvents: evidence.earlyFlakeRetryEvents,
|
|
313
|
-
taggedEvents: evidence.earlyFlakeTaggedEvents,
|
|
314
|
-
})
|
|
315
|
-
}
|
|
316
|
-
if (checkId === 'auto-test-retries') {
|
|
317
|
-
return compactDefined({
|
|
318
|
-
matchingTestEvents: evidence.matchingTestEvents,
|
|
319
|
-
retryEvents: evidence.autoTestRetryEvents,
|
|
320
|
-
failedAttempts: evidence.failedAttempts,
|
|
321
|
-
passedAttempts: evidence.passedAttempts,
|
|
322
|
-
})
|
|
323
|
-
}
|
|
324
|
-
if (checkId === 'test-management') {
|
|
325
|
-
return compactDefined({
|
|
326
|
-
matchingTestEvents: evidence.matchingTestEvents,
|
|
327
|
-
quarantinedEvents: evidence.quarantinedEvents,
|
|
177
|
+
/**
|
|
178
|
+
* Builds one plain-language verdict per framework.
|
|
179
|
+
*
|
|
180
|
+
* @param {object[]} results validation results
|
|
181
|
+
* @returns {string[]} Markdown bullets
|
|
182
|
+
*/
|
|
183
|
+
function getVerdicts (results) {
|
|
184
|
+
const grouped = groupByFramework(results)
|
|
185
|
+
const verdicts = []
|
|
186
|
+
for (const [framework, frameworkResults] of grouped) {
|
|
187
|
+
const basic = frameworkResults.find(result => result.scenario === 'basic-reporting')
|
|
188
|
+
const ci = frameworkResults.find(result => result.scenario === 'ci-wiring')
|
|
189
|
+
const advancedFinding = frameworkResults.find(result => {
|
|
190
|
+
return !['all', 'basic-reporting', 'ci-wiring'].includes(result.scenario) &&
|
|
191
|
+
['error', 'fail'].includes(result.status)
|
|
328
192
|
})
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
193
|
+
let text
|
|
194
|
+
if (basic?.status === 'pass') {
|
|
195
|
+
if (advancedFinding) {
|
|
196
|
+
const outcome = advancedFinding.status === 'fail' ? 'failed' : 'was incomplete'
|
|
197
|
+
text = `Basic Reporting passed, but ${getScenarioName(advancedFinding.scenario)} ${outcome}: ` +
|
|
198
|
+
advancedFinding.diagnosis
|
|
199
|
+
} else {
|
|
200
|
+
text = 'The library reported this project test correctly when initialized by the validator.'
|
|
201
|
+
}
|
|
202
|
+
if (ci?.status === 'fail') {
|
|
203
|
+
text += ' The customer CI configuration has a confirmed static problem.'
|
|
204
|
+
} else if (isIncomplete(ci)) {
|
|
205
|
+
text += ' The customer CI path is still unverified.'
|
|
206
|
+
}
|
|
207
|
+
} else if (basic?.evidence?.possibleLibraryBug) {
|
|
208
|
+
text = 'The clean test worked, but controlled Datadog initialization did not. This is a possible library bug ' +
|
|
209
|
+
'and the recorded debug artifacts are suitable for engineering investigation.'
|
|
210
|
+
} else if ((!basic || isIncomplete(basic)) && ci?.status === 'fail') {
|
|
211
|
+
text = 'The customer CI configuration has a confirmed static problem. Local library behavior was not ' +
|
|
212
|
+
'validated because the direct test or its environment was unavailable.'
|
|
213
|
+
} else if (isIncomplete(basic) || !basic) {
|
|
214
|
+
text = 'Local library behavior was not validated because the direct test or its environment was unavailable.'
|
|
215
|
+
} else {
|
|
216
|
+
text = basic.diagnosis
|
|
339
217
|
}
|
|
218
|
+
verdicts.push(`- **${plain(framework)}:** ${plain(text)}`)
|
|
340
219
|
}
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
const missing = []
|
|
345
|
-
if (events.sessions === 0) missing.push('test_session_end')
|
|
346
|
-
if (events.modules === 0) missing.push('test_module_end')
|
|
347
|
-
if (events.suites === 0) missing.push('test_suite_end')
|
|
348
|
-
if (events.tests === 0) missing.push('test')
|
|
349
|
-
return missing
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
function compactCiCommandCandidate (candidate, repositoryRoot) {
|
|
353
|
-
if (!candidate) return
|
|
354
|
-
|
|
355
|
-
return {
|
|
356
|
-
provider: candidate.provider,
|
|
357
|
-
configFile: relativeRepositoryPath(candidate.configFile, repositoryRoot),
|
|
358
|
-
workflow: candidate.workflow,
|
|
359
|
-
job: candidate.job,
|
|
360
|
-
step: candidate.step,
|
|
361
|
-
command: candidate.command,
|
|
362
|
-
whySelected: candidate.whySelected,
|
|
363
|
-
}
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
function compactInitializationProbe (probe) {
|
|
367
|
-
if (!probe) return
|
|
368
|
-
|
|
369
|
-
return compactDefined({
|
|
370
|
-
ran: probe.ran,
|
|
371
|
-
processCount: probe.processCount,
|
|
372
|
-
reachedAnyNodeProcess: probe.reachedAnyNodeProcess,
|
|
373
|
-
reachedTestRunnerProcess: probe.reachedTestRunnerProcess,
|
|
374
|
-
})
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
function getEventCounts (evidence) {
|
|
378
|
-
const events = compactDefined({
|
|
379
|
-
sessions: evidence.testSessionEvents,
|
|
380
|
-
modules: evidence.testModuleEvents,
|
|
381
|
-
suites: evidence.testSuiteEvents,
|
|
382
|
-
tests: evidence.testEvents,
|
|
383
|
-
})
|
|
384
|
-
return Object.keys(events).length > 0 ? events : undefined
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
function compactArtifacts (artifacts, reportDirectory) {
|
|
388
|
-
const compact = {}
|
|
389
|
-
for (const [name, artifactPath] of getKeyArtifacts(artifacts)) {
|
|
390
|
-
if (!artifactPath) continue
|
|
391
|
-
compact[toCamelCase(name)] = relativeArtifactPath(artifactPath, reportDirectory)
|
|
392
|
-
}
|
|
393
|
-
return compact
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
function getRelativeArtifactDirectory (artifacts, reportDirectory) {
|
|
397
|
-
if (!Array.isArray(artifacts) || artifacts.length === 0) return
|
|
398
|
-
return relativeArtifactPath(getCommonArtifactDirectory(artifacts), reportDirectory)
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
function relativeArtifactPath (artifactPath, reportDirectory) {
|
|
402
|
-
if (!path.isAbsolute(artifactPath)) return artifactPath.split(path.sep).join('/').replace(/\/$/, '') || '.'
|
|
403
|
-
return path.relative(reportDirectory, artifactPath).split(path.sep).join('/') || '.'
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
function relativeRepositoryPath (value, repositoryRoot) {
|
|
407
|
-
if (!value || !repositoryRoot || !path.isAbsolute(value)) return value
|
|
408
|
-
const relative = path.relative(repositoryRoot, value)
|
|
409
|
-
return relative && !relative.startsWith('..') && !path.isAbsolute(relative)
|
|
410
|
-
? relative.split(path.sep).join('/')
|
|
411
|
-
: value
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
function compactDefined (value) {
|
|
415
|
-
const compact = {}
|
|
416
|
-
for (const [key, entry] of Object.entries(value)) {
|
|
417
|
-
if (entry !== undefined) compact[key] = entry
|
|
418
|
-
}
|
|
419
|
-
return compact
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
function toCamelCase (value) {
|
|
423
|
-
return value.charAt(0).toLowerCase() + value.slice(1).replaceAll(/\s+(.)/g, (_, character) => {
|
|
424
|
-
return character.toUpperCase()
|
|
425
|
-
})
|
|
220
|
+
return verdicts.length > 0
|
|
221
|
+
? verdicts
|
|
222
|
+
: ['- No live framework check completed. The validation is incomplete.']
|
|
426
223
|
}
|
|
427
224
|
|
|
428
225
|
/**
|
|
429
|
-
*
|
|
226
|
+
* Returns actionable recommendations without duplicating them.
|
|
430
227
|
*
|
|
431
|
-
* @param {
|
|
432
|
-
* @
|
|
433
|
-
* @returns {string} inert Markdown text
|
|
228
|
+
* @param {object[]} results validation results
|
|
229
|
+
* @returns {Array<{framework: string, check: string, text: string}>} actions
|
|
434
230
|
*/
|
|
435
|
-
function
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
231
|
+
function getActions (results) {
|
|
232
|
+
const actions = []
|
|
233
|
+
const seen = new Set()
|
|
234
|
+
for (const result of results) {
|
|
235
|
+
if (result.status === 'pass') continue
|
|
236
|
+
const recommendation = findRecommendation(result.evidence) || getFallbackAction(result)
|
|
237
|
+
if (!recommendation) continue
|
|
238
|
+
const key = `${result.frameworkId}:${result.scenario}:${recommendation}`
|
|
239
|
+
if (seen.has(key)) continue
|
|
240
|
+
seen.add(key)
|
|
241
|
+
actions.push({
|
|
242
|
+
check: getScenarioName(result.scenario),
|
|
243
|
+
framework: result.frameworkDisplayName,
|
|
244
|
+
text: recommendation,
|
|
245
|
+
})
|
|
447
246
|
}
|
|
448
|
-
|
|
449
|
-
return String(value ?? '')
|
|
450
|
-
.replaceAll(/\r?\n/g, ' ')
|
|
451
|
-
.replaceAll('\\', '\\\\')
|
|
452
|
-
.replace(/^(\s{0,3})>/, String.raw`$1\>`)
|
|
453
|
-
.replace(/^(\s{0,3})(#{1,6}|-{1,3}|\+|~{3,})(?=\s|$)/, String.raw`$1\$2`)
|
|
454
|
-
.replace(/^(\s{0,3}\d+)([.)])(?=\s|$)/, String.raw`$1\$2`)
|
|
455
|
-
.replaceAll('<', String.raw`\<`)
|
|
456
|
-
.replaceAll(/([`!*_[\]()|])/g, String.raw`\$1`)
|
|
247
|
+
return actions.slice(0, 12)
|
|
457
248
|
}
|
|
458
249
|
|
|
459
250
|
/**
|
|
460
|
-
*
|
|
251
|
+
* Finds the first explicit recommendation in an evidence tree.
|
|
461
252
|
*
|
|
462
|
-
* @param {unknown} value
|
|
463
|
-
* @returns {string}
|
|
253
|
+
* @param {unknown} value evidence value
|
|
254
|
+
* @returns {string|undefined} recommendation
|
|
464
255
|
*/
|
|
465
|
-
function
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
return `\`${content}\``
|
|
256
|
+
function findRecommendation (value) {
|
|
257
|
+
if (!value || typeof value !== 'object') return
|
|
258
|
+
if (typeof value.recommendation === 'string' && value.recommendation.trim()) return value.recommendation
|
|
259
|
+
for (const child of Object.values(value)) {
|
|
260
|
+
const recommendation = findRecommendation(child)
|
|
261
|
+
if (recommendation) return recommendation
|
|
262
|
+
}
|
|
473
263
|
}
|
|
474
264
|
|
|
475
265
|
/**
|
|
476
|
-
*
|
|
266
|
+
* Returns a useful fallback action.
|
|
477
267
|
*
|
|
478
|
-
* @param {
|
|
479
|
-
* @returns {string}
|
|
268
|
+
* @param {object} result validation result
|
|
269
|
+
* @returns {string|undefined} fallback
|
|
480
270
|
*/
|
|
481
|
-
function
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
const code = character.charCodeAt(0)
|
|
486
|
-
result += code <= 0x1F || code === 0x7F ? ' ' : character
|
|
271
|
+
function getFallbackAction (result) {
|
|
272
|
+
if (result.frameworkId === 'validator' || result.frameworkId === 'validation-cleanup') {
|
|
273
|
+
return 'Keep the validation artifacts and report this validator failure to engineering. Project setup changes ' +
|
|
274
|
+
'will not resolve it.'
|
|
487
275
|
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
function getCiCommandCandidates (manifest, frameworkLabels = getFrameworkLabels(manifest)) {
|
|
492
|
-
const candidates = []
|
|
493
|
-
|
|
494
|
-
for (const framework of manifest.frameworks || []) {
|
|
495
|
-
const candidate = buildCiCommandCandidate(framework)
|
|
496
|
-
if (!candidate) continue
|
|
497
|
-
candidates.push({
|
|
498
|
-
frameworkId: framework.id,
|
|
499
|
-
frameworkDisplayName: frameworkLabels.get(framework.id) || framework.id,
|
|
500
|
-
...candidate,
|
|
501
|
-
})
|
|
276
|
+
if (result.scenario === 'ci-wiring') {
|
|
277
|
+
return 'Resolve the exact CI test job and effective environment, or rerun it with DD_TRACE_DEBUG=1.'
|
|
502
278
|
}
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
function getStringArray (values) {
|
|
508
|
-
if (!Array.isArray(values)) return []
|
|
509
|
-
return values.filter(value => typeof value === 'string')
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
function getStaticDiagnosisNotes (diagnosis, validationResults) {
|
|
513
|
-
const diagnosisResults = Array.isArray(diagnosis?.results) ? diagnosis.results : []
|
|
514
|
-
const notes = []
|
|
515
|
-
const conclusiveCiWiring = getCiWiringResults(validationResults).some(result => {
|
|
516
|
-
return !isIncompleteResult(result) && (result.status === 'pass' || result.status === 'fail')
|
|
517
|
-
})
|
|
518
|
-
|
|
519
|
-
if (diagnosisResults.some(isMissingStaticInitializationResult) &&
|
|
520
|
-
getLiveValidationResults(validationResults).length === 0) {
|
|
521
|
-
notes.push(
|
|
522
|
-
'Static diagnosis found no Test Optimization initialization in the inspected CI configuration, but no live ' +
|
|
523
|
-
'test command ran. Treat this as context only, not as a confirmed CI-wiring failure or remediation. First ' +
|
|
524
|
-
'make one representative test command runnable and rerun validation.'
|
|
525
|
-
)
|
|
526
|
-
} else if (diagnosisResults.some(isMissingStaticInitializationResult) && !conclusiveCiWiring) {
|
|
527
|
-
notes.push(
|
|
528
|
-
'Static diagnosis found no Test Optimization initialization in the inspected CI configuration, but the ' +
|
|
529
|
-
'selected CI replay did not reach a test result. Treat this as context only, not as a confirmed CI-wiring ' +
|
|
530
|
-
'failure or remediation. Correct the replay command and rerun validation first.'
|
|
531
|
-
)
|
|
532
|
-
} else if (diagnosisResults.some(isMissingStaticInitializationResult)) {
|
|
533
|
-
notes.push(
|
|
534
|
-
'Static diagnosis reported missing NODE_OPTIONS/dd-trace/ci/init. In this validation report, that is a ' +
|
|
535
|
-
'CI wiring/static configuration finding, not a direct-initialization Basic Reporting blocker.'
|
|
536
|
-
)
|
|
279
|
+
if (result.evidence?.possibleLibraryBug) {
|
|
280
|
+
return 'Send the debug and clean-run artifacts, framework version, and dd-trace version to engineering.'
|
|
537
281
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
}
|
|
541
|
-
|
|
542
|
-
function isMissingStaticInitializationResult (result) {
|
|
543
|
-
return result?.title === 'Missing Test Optimization initialization' ||
|
|
544
|
-
result?.title === 'CI workflows do not show Test Optimization initialization'
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
function appendMarkdownCiDiscovery (lines, ciDiscovery) {
|
|
548
|
-
if (!ciDiscovery) return
|
|
549
|
-
|
|
550
|
-
lines.push('## CI Configuration Inspected', '')
|
|
551
|
-
appendMarkdownList(lines, 'Workflow files', ciDiscovery.found)
|
|
552
|
-
appendMarkdownList(lines, 'Warnings', ciDiscovery.warnings)
|
|
553
|
-
appendMarkdownList(lines, 'Contradictions', ciDiscovery.contradictions)
|
|
554
|
-
lines.push('')
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
function appendMarkdownStaticDiagnosisNotes (lines, notes) {
|
|
558
|
-
if (!Array.isArray(notes) || notes.length === 0) return
|
|
559
|
-
|
|
560
|
-
lines.push('## Static Diagnosis Notes', '')
|
|
561
|
-
for (const note of notes) {
|
|
562
|
-
lines.push(`- ${markdownText(note)}`)
|
|
282
|
+
if (isIncomplete(result)) {
|
|
283
|
+
return 'Prepare the project so the selected direct test passes normally, then create a fresh validation plan.'
|
|
563
284
|
}
|
|
564
|
-
lines.push('')
|
|
565
285
|
}
|
|
566
286
|
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
287
|
+
/**
|
|
288
|
+
* Retains high-signal structured evidence for failed or incomplete checks.
|
|
289
|
+
*
|
|
290
|
+
* @param {object} evidence full evidence
|
|
291
|
+
* @returns {object} compact evidence
|
|
292
|
+
*/
|
|
293
|
+
function compactEvidence (evidence = {}) {
|
|
294
|
+
const keys = [
|
|
295
|
+
'commandExitCode',
|
|
296
|
+
'commandFailure',
|
|
297
|
+
'commandOutputSummary',
|
|
298
|
+
'commandTimedOut',
|
|
299
|
+
'conclusion',
|
|
300
|
+
'domain',
|
|
301
|
+
'evidenceStrength',
|
|
302
|
+
'foundationalReportingEstablished',
|
|
303
|
+
'missingEventLevels',
|
|
304
|
+
'offlineExporterInitialized',
|
|
305
|
+
'possibleLibraryBug',
|
|
306
|
+
'preflight',
|
|
307
|
+
'recommendation',
|
|
308
|
+
'reportingPath',
|
|
309
|
+
'settingsLoadedFromCache',
|
|
310
|
+
'testEvents',
|
|
311
|
+
'testModuleEvents',
|
|
312
|
+
'testSessionEvents',
|
|
313
|
+
'testSuiteEvents',
|
|
314
|
+
'validationIncomplete',
|
|
315
|
+
]
|
|
316
|
+
const compact = {}
|
|
317
|
+
for (const key of keys) {
|
|
318
|
+
if (evidence[key] !== undefined) compact[key] = evidence[key]
|
|
579
319
|
}
|
|
580
|
-
|
|
320
|
+
if (evidence.ciCommandCandidate) compact.ciCommandCandidate = evidence.ciCommandCandidate
|
|
321
|
+
if (evidence.ciFacts) compact.ciFacts = evidence.ciFacts
|
|
322
|
+
if (evidence.ciWiring?.unresolved) compact.unresolved = evidence.ciWiring.unresolved
|
|
323
|
+
return sanitizeForReport(compact)
|
|
581
324
|
}
|
|
582
325
|
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
markdownText(formatScenarioName(result.scenario)),
|
|
592
|
-
''
|
|
593
|
-
)
|
|
594
|
-
if (result.scenario !== CI_WIRING_SCENARIO) {
|
|
595
|
-
lines.push(`Evidence conclusion: ${markdownText(result.diagnosis, { preserveInlineCode: true })}`, '')
|
|
596
|
-
}
|
|
597
|
-
for (const detail of getResultDetailLines(result, { markdown: true })) {
|
|
598
|
-
lines.push(`- ${markdownText(detail, { preserveInlineCode: true })}`)
|
|
599
|
-
}
|
|
600
|
-
if (Array.isArray(result.artifacts) && result.artifacts.length > 0) {
|
|
601
|
-
const directory = getCommonArtifactDirectory(result.artifacts)
|
|
602
|
-
const relative = path.relative(reportDirectory, directory).split(path.sep).join('/') || '.'
|
|
603
|
-
lines.push(`- Scenario artifacts: [open artifact directory](<${relative}/>)`)
|
|
604
|
-
}
|
|
605
|
-
lines.push('')
|
|
606
|
-
}
|
|
326
|
+
/**
|
|
327
|
+
* Renders JSON without allowing evidence strings to close the Markdown fence.
|
|
328
|
+
*
|
|
329
|
+
* @param {object} value structured evidence
|
|
330
|
+
* @returns {string} fenced JSON body
|
|
331
|
+
*/
|
|
332
|
+
function fencedJson (value) {
|
|
333
|
+
return JSON.stringify(value, null, 2).replaceAll('```', String.raw`\u0060\u0060\u0060`)
|
|
607
334
|
}
|
|
608
335
|
|
|
609
336
|
/**
|
|
610
|
-
*
|
|
337
|
+
* Renders the compact console summary.
|
|
611
338
|
*
|
|
612
|
-
* @param {string[]} lines rendered report lines
|
|
613
339
|
* @param {object[]} results validation results
|
|
614
|
-
* @
|
|
340
|
+
* @param {object} runSummary run summary
|
|
341
|
+
* @param {string} reportPath report path
|
|
342
|
+
* @returns {string} console text
|
|
615
343
|
*/
|
|
616
|
-
function
|
|
617
|
-
const
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
`### ${markdownText(entry.frameworkDisplayName)}: ${markdownText(formatScenarioName(entry.scenario))}`,
|
|
624
|
-
''
|
|
625
|
-
)
|
|
626
|
-
for (const recommendation of entry.recommendations) {
|
|
627
|
-
lines.push(`- ${markdownText(recommendation, { preserveInlineCode: true })}`)
|
|
628
|
-
}
|
|
629
|
-
appendMarkdownCiRemediation(lines, entry.ciRemediation)
|
|
630
|
-
lines.push('')
|
|
631
|
-
}
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
function appendMarkdownCiRemediation (lines, remediation) {
|
|
635
|
-
if (!remediation?.variants?.length) return
|
|
636
|
-
|
|
637
|
-
for (const variant of remediation.variants) {
|
|
344
|
+
function renderConsole (results, runSummary, reportPath) {
|
|
345
|
+
const lines = [
|
|
346
|
+
`Test Optimization validation: ${formatExecutionStatus(runSummary.executionStatus)}`,
|
|
347
|
+
`Coverage: ${runSummary.validationCoverage === 'complete' ? 'complete' : 'partial'}`,
|
|
348
|
+
`Cleanup: ${formatCleanupStatus(runSummary.cleanup)}`,
|
|
349
|
+
]
|
|
350
|
+
for (const result of getVisibleResults(results)) {
|
|
638
351
|
lines.push(
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
'',
|
|
642
|
-
`Required: ${markdownText(variant.prerequisite)}`,
|
|
643
|
-
'',
|
|
644
|
-
`Required variables: ${variant.requiredValues.map(value => {
|
|
645
|
-
const source = value.source === 'ci-secret-store' ? ' (value from CI secret store)' : ''
|
|
646
|
-
return `${markdownCode(value.name)}${source}`
|
|
647
|
-
}).join(', ')}`,
|
|
648
|
-
'',
|
|
649
|
-
`Recommended variables: ${(variant.recommendedValues || []).map(value => {
|
|
650
|
-
return `${markdownCode(`${value.name}=${value.value}`)} (${markdownText(value.description)})`
|
|
651
|
-
}).join(', ') || 'none.'}`,
|
|
652
|
-
'',
|
|
653
|
-
`Optional variables: ${(variant.optionalValues || []).map(value => {
|
|
654
|
-
return `${markdownCode(value.name)} (${markdownText(value.description)})`
|
|
655
|
-
}).join(', ') || 'none for this minimal setup.'}`,
|
|
656
|
-
'',
|
|
657
|
-
variant.snippet.includes('env:') ? '```yaml' : '```text',
|
|
658
|
-
variant.snippet.replaceAll('```', String.raw`\u0060\u0060\u0060`),
|
|
659
|
-
'```'
|
|
352
|
+
`${getDisplayStatus(result)} ${result.frameworkDisplayName} / ${getScenarioName(result.scenario)}: ` +
|
|
353
|
+
plain(result.diagnosis)
|
|
660
354
|
)
|
|
661
355
|
}
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
function appendMarkdownList (lines, label, values) {
|
|
665
|
-
if (!Array.isArray(values) || values.length === 0) return
|
|
666
|
-
lines.push(`- ${label}: ${values.map(markdownCode).join(', ')}`)
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
function appendMarkdownJsonSection (lines, title, value) {
|
|
670
|
-
if (value === undefined) return
|
|
671
|
-
|
|
672
|
-
const json = JSON.stringify(value, null, 2).replaceAll('```', String.raw`\u0060\u0060\u0060`)
|
|
673
|
-
lines.push(
|
|
674
|
-
'',
|
|
675
|
-
`<details><summary>${title}</summary>`,
|
|
676
|
-
'',
|
|
677
|
-
'```json',
|
|
678
|
-
json,
|
|
679
|
-
'```',
|
|
680
|
-
'',
|
|
681
|
-
'</details>'
|
|
682
|
-
)
|
|
683
|
-
}
|
|
684
|
-
|
|
685
|
-
function formatCiCommandCandidateSummary (candidate, options = {}) {
|
|
686
|
-
const format = options.markdown
|
|
687
|
-
? markdownCode
|
|
688
|
-
: value => value
|
|
689
|
-
const parts = [
|
|
690
|
-
candidate.provider && format(candidate.provider),
|
|
691
|
-
candidate.workflow && `workflow ${format(candidate.workflow)}`,
|
|
692
|
-
candidate.job && `job ${format(candidate.job)}`,
|
|
693
|
-
candidate.step && `step ${format(candidate.step)}`,
|
|
694
|
-
candidate.command && `command ${format(candidate.command)}`,
|
|
695
|
-
candidate.cwd && `cwd ${format(candidate.cwd)}`,
|
|
696
|
-
].filter(Boolean)
|
|
697
|
-
|
|
698
|
-
return parts.length > 0 ? parts.join('; ') : 'CI command metadata was not determined'
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
function formatCiCommandCandidateDetails (candidate, options = {}) {
|
|
702
|
-
const details = []
|
|
703
|
-
const format = options.markdown
|
|
704
|
-
? markdownCode
|
|
705
|
-
: value => value
|
|
706
|
-
|
|
707
|
-
if (candidate.whySelected) {
|
|
708
|
-
details.push(`Selected because: ${candidate.whySelected}`)
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
const envSummary = formatCiEnvSummary(candidate.env, { format })
|
|
712
|
-
if (envSummary) {
|
|
713
|
-
details.push(`Environment found in CI: ${envSummary}`)
|
|
714
|
-
}
|
|
715
|
-
|
|
716
|
-
const expansion = formatChain(candidate.packageScriptExpansionChain, { format })
|
|
717
|
-
if (expansion) {
|
|
718
|
-
details.push(`Package script expansion: ${expansion}`)
|
|
719
|
-
}
|
|
720
|
-
|
|
721
|
-
const toolChain = formatChain(candidate.runnerToolChain, { format })
|
|
722
|
-
if (toolChain) {
|
|
723
|
-
details.push(`Runner/tool chain: ${toolChain}`)
|
|
724
|
-
}
|
|
725
|
-
|
|
726
|
-
const setupCommands = formatChain(candidate.setupCommandIds, { format })
|
|
727
|
-
if (setupCommands) {
|
|
728
|
-
details.push(`Required setup command ids: ${setupCommands}`)
|
|
729
|
-
}
|
|
730
|
-
|
|
731
|
-
const unresolved = formatChain(candidate.unresolved, { format })
|
|
732
|
-
if (unresolved) {
|
|
733
|
-
details.push(`Unresolved replay details: ${unresolved}`)
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
const commandDetails = formatCommandDetails(candidate.commandDetails)
|
|
737
|
-
if (commandDetails) {
|
|
738
|
-
details.push(`Command display details: ${commandDetails}`)
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
return details
|
|
742
|
-
}
|
|
743
|
-
|
|
744
|
-
function formatCiEnvSummary (env, { format }) {
|
|
745
|
-
if (!env || typeof env !== 'object') return ''
|
|
746
|
-
|
|
747
|
-
const parts = []
|
|
748
|
-
for (const scope of ['workflow', 'job', 'step', 'inherited']) {
|
|
749
|
-
const values = formatEnvPairs(env[scope], { format })
|
|
750
|
-
if (values) parts.push(`${scope} ${values}`)
|
|
751
|
-
}
|
|
752
|
-
return parts.join('; ')
|
|
753
|
-
}
|
|
754
|
-
|
|
755
|
-
function formatEnvPairs (env, { format }) {
|
|
756
|
-
if (!env || typeof env !== 'object') return ''
|
|
757
|
-
|
|
758
|
-
const pairs = []
|
|
759
|
-
for (const [name, value] of Object.entries(env)) {
|
|
760
|
-
pairs.push(format(`${name}=${value}`))
|
|
761
|
-
}
|
|
762
|
-
return pairs.join(', ')
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
function formatChain (values, { format }) {
|
|
766
|
-
if (!Array.isArray(values) || values.length === 0) return ''
|
|
767
|
-
return values.map(value => format(value)).join(' -> ')
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
function formatCommandDetails (details) {
|
|
771
|
-
if (!details || typeof details !== 'object') return ''
|
|
772
|
-
|
|
773
|
-
const parts = []
|
|
774
|
-
if (details.runtimeWrapper) parts.push(`runtime wrapper ${details.runtimeWrapper}`)
|
|
775
|
-
if (details.packageManager) parts.push(`package manager ${details.packageManager}`)
|
|
776
|
-
if (details.pathAdjusted) parts.push('PATH adjusted')
|
|
777
|
-
if (details.exactCommandCollapsed) parts.push('display command collapsed runtime plumbing')
|
|
778
|
-
return parts.join('; ')
|
|
779
|
-
}
|
|
780
|
-
|
|
781
|
-
function shouldRenderResultDetails (result) {
|
|
782
|
-
return result.status === 'fail' || result.status === 'error' || result.status === 'blocked'
|
|
783
|
-
}
|
|
784
|
-
|
|
785
|
-
function getResultDetailLines (result, options = {}) {
|
|
786
|
-
const evidence = result.evidence || {}
|
|
787
|
-
const format = options.markdown
|
|
788
|
-
? markdownCode
|
|
789
|
-
: value => value
|
|
790
|
-
const lines = []
|
|
791
|
-
const command = readResultCommand(result) || getEvidenceCommand(evidence)
|
|
792
|
-
|
|
793
|
-
if (command?.command) lines.push(`Command: ${format(command.command)}`)
|
|
794
|
-
if (command?.cwd) lines.push(`Cwd: ${format(command.cwd)}`)
|
|
795
|
-
if (command?.exitCode !== undefined) lines.push(`Exit code: ${format(command.exitCode)}`)
|
|
796
|
-
if (command?.timedOut !== undefined) lines.push(`Timed out: ${format(command.timedOut)}`)
|
|
797
|
-
if (command?.durationMs !== undefined) lines.push(`Duration ms: ${format(command.durationMs)}`)
|
|
798
|
-
|
|
799
|
-
if (Array.isArray(evidence.commandOutputSummary) && evidence.commandOutputSummary.length > 0) {
|
|
800
|
-
lines.push(`Command output summary: ${formatList(evidence.commandOutputSummary, { format })}`)
|
|
801
|
-
}
|
|
802
|
-
if (Array.isArray(evidence.existingDatadogInitScripts) && evidence.existingDatadogInitScripts.length > 0) {
|
|
803
|
-
const scripts = evidence.existingDatadogInitScripts.map(script => {
|
|
804
|
-
return `${script.name} (${script.packageJson})`
|
|
805
|
-
})
|
|
806
|
-
lines.push(`Existing package scripts with Datadog initialization: ${formatList(scripts, { format })}`)
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
if (evidence.reason) lines.push(`Reason: ${evidence.reason}`)
|
|
810
|
-
if (evidence.error) lines.push(`Error: ${format(evidence.error)}`)
|
|
811
|
-
if (evidence.errorCode) lines.push(`Error code: ${format(evidence.errorCode)}`)
|
|
812
|
-
if (evidence.errorSyscall) lines.push(`Error syscall: ${format(evidence.errorSyscall)}`)
|
|
813
|
-
if (evidence.errorAddress) lines.push(`Error address: ${format(evidence.errorAddress)}`)
|
|
814
|
-
if (evidence.projectCommandsRan !== undefined) {
|
|
815
|
-
lines.push(`Project commands ran: ${format(evidence.projectCommandsRan)}`)
|
|
816
|
-
}
|
|
817
|
-
if (evidence.workingDirectory) lines.push(`Host working directory: ${format(evidence.workingDirectory)}`)
|
|
818
|
-
if (evidence.approvedPlanSha256) lines.push(`Approved plan digest: ${format(evidence.approvedPlanSha256)}`)
|
|
819
|
-
if (Array.isArray(evidence.remediation) && evidence.remediation.length > 0) {
|
|
820
|
-
lines.push(`Remediation: ${formatList(evidence.remediation, { format })}`)
|
|
821
|
-
}
|
|
822
|
-
if (evidence.rerunCommand) lines.push(`Rerun command: ${format(evidence.rerunCommand)}`)
|
|
823
|
-
|
|
824
|
-
appendExcerptLine(lines, 'Stdout excerpt', evidence.commandFailure?.stdoutExcerpt, { format })
|
|
825
|
-
appendExcerptLine(lines, 'Stderr excerpt', evidence.commandFailure?.stderrExcerpt, { format })
|
|
826
|
-
if (evidence.commandFailure?.summary) {
|
|
827
|
-
lines.push(`Command failure: ${evidence.commandFailure.summary}`)
|
|
828
|
-
}
|
|
829
|
-
if (evidence.commandFailure?.recommendation) {
|
|
830
|
-
lines.push(`Command failure recommendation: ${evidence.commandFailure.recommendation}`)
|
|
831
|
-
}
|
|
832
|
-
appendExcerptLine(lines, 'Command failure signals', evidence.commandFailure?.signals, { format })
|
|
833
|
-
appendExcerptLine(lines, 'Command build/setup errors', evidence.commandFailure?.buildErrors, { format })
|
|
834
|
-
appendExcerptLine(lines, 'CI debug lines', evidence.debugSignals?.lines, { format })
|
|
835
|
-
appendExcerptLine(lines, 'Debug lines', evidence.debugRerun?.debugLines, { format })
|
|
836
|
-
appendExcerptLine(lines, 'Debug stdout excerpt', evidence.debugRerun?.stdoutExcerpt, { format })
|
|
837
|
-
appendExcerptLine(lines, 'Debug stderr excerpt', evidence.debugRerun?.stderrExcerpt, { format })
|
|
838
|
-
appendSetupFailureLines(lines, evidence, { format })
|
|
839
|
-
appendEventFailureLines(lines, evidence, { format })
|
|
840
|
-
appendInitializationProbeLines(lines, evidence.initializationProbe, { format })
|
|
841
|
-
appendMonorepoFindingLines(lines, evidence.monorepoFindings, { format })
|
|
842
|
-
|
|
843
|
-
return lines.length > 0 ? lines : ['No additional structured evidence was recorded.']
|
|
844
|
-
}
|
|
845
|
-
|
|
846
|
-
function getCommonArtifactDirectory (artifacts) {
|
|
847
|
-
let directory = path.dirname(path.resolve(artifacts[0]))
|
|
848
|
-
while (!artifacts.every(artifact => isPathInside(directory, path.resolve(artifact)))) {
|
|
849
|
-
const parent = path.dirname(directory)
|
|
850
|
-
if (parent === directory) return directory
|
|
851
|
-
directory = parent
|
|
852
|
-
}
|
|
853
|
-
return directory
|
|
854
|
-
}
|
|
855
|
-
|
|
856
|
-
function isPathInside (directory, filename) {
|
|
857
|
-
const relative = path.relative(directory, filename)
|
|
858
|
-
return relative === '' || (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative))
|
|
859
|
-
}
|
|
860
|
-
|
|
861
|
-
function relativizeHumanLines (lines, repositoryRoot) {
|
|
862
|
-
if (!repositoryRoot) return
|
|
863
|
-
const absoluteRoot = path.resolve(repositoryRoot)
|
|
864
|
-
const rootWithSeparator = `${absoluteRoot}${path.sep}`
|
|
865
|
-
for (let index = 0; index < lines.length; index++) {
|
|
866
|
-
lines[index] = lines[index]
|
|
867
|
-
.replaceAll(rootWithSeparator, '')
|
|
868
|
-
.replaceAll(absoluteRoot, '.')
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
|
|
872
|
-
function readResultCommand (result) {
|
|
873
|
-
const commandArtifact = (result.artifacts || []).find(artifact => path.basename(artifact) === 'command.json')
|
|
874
|
-
if (!commandArtifact) return
|
|
875
|
-
|
|
876
|
-
try {
|
|
877
|
-
const artifact = JSON.parse(fs.readFileSync(commandArtifact, 'utf8'))
|
|
878
|
-
return {
|
|
879
|
-
command: sanitizeString(artifact.displayCommand || artifact.command),
|
|
880
|
-
cwd: artifact.cwd,
|
|
881
|
-
durationMs: artifact.durationMs,
|
|
882
|
-
exitCode: artifact.exitCode,
|
|
883
|
-
timedOut: artifact.timedOut,
|
|
884
|
-
}
|
|
885
|
-
} catch {}
|
|
886
|
-
}
|
|
887
|
-
|
|
888
|
-
function getEvidenceCommand (evidence) {
|
|
889
|
-
const setupCommand = evidence.setupCommand
|
|
890
|
-
if (!setupCommand) return
|
|
891
|
-
|
|
892
|
-
return {
|
|
893
|
-
command: sanitizeString(setupCommand.command),
|
|
894
|
-
cwd: setupCommand.cwd,
|
|
895
|
-
exitCode: setupCommand.exitCode,
|
|
896
|
-
timedOut: setupCommand.timedOut,
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
|
|
900
|
-
function appendExcerptLine (lines, label, values, { format }) {
|
|
901
|
-
if (!Array.isArray(values) || values.length === 0) return
|
|
902
|
-
lines.push(`${label}: ${formatList(values, { format })}`)
|
|
903
|
-
}
|
|
904
|
-
|
|
905
|
-
function appendSetupFailureLines (lines, evidence, { format }) {
|
|
906
|
-
const setupCommand = evidence.setupCommand
|
|
907
|
-
if (!setupCommand) return
|
|
908
|
-
|
|
909
|
-
lines.push(`Setup failed: ${setupCommand.description || setupCommand.id || setupCommand.command}`)
|
|
910
|
-
if (setupCommand.stdoutSummary) {
|
|
911
|
-
lines.push(`Setup stdout excerpt: ${format(setupCommand.stdoutSummary)}`)
|
|
912
|
-
}
|
|
913
|
-
if (setupCommand.stderrSummary) {
|
|
914
|
-
lines.push(`Setup stderr excerpt: ${format(setupCommand.stderrSummary)}`)
|
|
915
|
-
}
|
|
916
|
-
}
|
|
917
|
-
|
|
918
|
-
function appendEventFailureLines (lines, evidence, { format }) {
|
|
919
|
-
const failure = evidence.eventLevelFailure
|
|
920
|
-
if (!failure) return
|
|
921
|
-
|
|
922
|
-
if (failure.kind) lines.push(`Event failure kind: ${format(failure.kind)}`)
|
|
923
|
-
if (Array.isArray(failure.missingLevels) && failure.missingLevels.length > 0) {
|
|
924
|
-
lines.push(`Missing event levels: ${formatList(failure.missingLevels, { format })}`)
|
|
925
|
-
}
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
function appendInitializationProbeLines (lines, probe, { format }) {
|
|
929
|
-
if (!probe) return
|
|
930
|
-
if (probe.ran !== true) {
|
|
931
|
-
if (probe.skippedBecauseConfigurationProvesRemoval) {
|
|
932
|
-
lines.push(`NODE_OPTIONS probe: not needed; ${probe.reason}`)
|
|
933
|
-
}
|
|
934
|
-
return
|
|
935
|
-
}
|
|
936
|
-
|
|
937
|
-
lines.push(`NODE_OPTIONS probe: reached Node process ${format(probe.reachedAnyNodeProcess)}, ` +
|
|
938
|
-
`reached test runner ${format(probe.reachedTestRunnerProcess)}, processes ${format(probe.processCount || 0)}`)
|
|
939
|
-
if (probe.stoppedAfterRunnerReached) {
|
|
940
|
-
lines.push('Probe execution: stopped immediately after the selected test runner was reached')
|
|
941
|
-
}
|
|
942
|
-
appendToolSignalLine(lines, 'Probe test runner signals', probe.testRunnerSignals, { format })
|
|
943
|
-
appendToolSignalLine(lines, 'Probe wrapper signals', probe.wrapperSignals, { format })
|
|
944
|
-
appendToolSignalLine(lines, 'Probe package manager signals', probe.packageManagerSignals, { format })
|
|
945
|
-
if (probe.recordsPath) lines.push(`Probe records: ${format(probe.recordsPath)}`)
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
function appendToolSignalLine (lines, label, signals, { format }) {
|
|
949
|
-
if (!Array.isArray(signals) || signals.length === 0) return
|
|
950
|
-
|
|
951
|
-
const values = signals.map(signal => {
|
|
952
|
-
const processCount = signal.processCount || (signal.pid ? 1 : 0)
|
|
953
|
-
const parts = [
|
|
954
|
-
signal.name,
|
|
955
|
-
processCount && `${processCount} process${processCount === 1 ? '' : 'es'}`,
|
|
956
|
-
signal.cwd && `cwd ${signal.cwd}`,
|
|
957
|
-
].filter(Boolean)
|
|
958
|
-
return parts.join(' ')
|
|
959
|
-
})
|
|
960
|
-
lines.push(`${label}: ${formatList(values, { format })}`)
|
|
961
|
-
}
|
|
962
|
-
|
|
963
|
-
function appendMonorepoFindingLines (lines, findings, { format }) {
|
|
964
|
-
if (!Array.isArray(findings) || findings.length === 0) return
|
|
965
|
-
|
|
966
|
-
for (const finding of findings) {
|
|
967
|
-
const parts = [
|
|
968
|
-
finding.id,
|
|
969
|
-
finding.tool && `tool ${finding.tool}`,
|
|
970
|
-
finding.reason,
|
|
971
|
-
finding.recommendation && `Recommendation: ${finding.recommendation}`,
|
|
972
|
-
].filter(Boolean)
|
|
973
|
-
lines.push(`Monorepo finding: ${formatList(parts, { format })}`)
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
function summarizeOmittedCommands (commands) {
|
|
978
|
-
const groups = new Map()
|
|
979
|
-
for (const command of commands) {
|
|
980
|
-
const category = getOmittedCommandCategory(command)
|
|
981
|
-
const group = groups.get(category.id) || { ...category, count: 0 }
|
|
982
|
-
group.count++
|
|
983
|
-
groups.set(category.id, group)
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
return [...groups.values()].map(group => {
|
|
987
|
-
const count = group.count === 1 ? '1 command' : `${group.count} commands`
|
|
988
|
-
return `${group.label} (${count}): ${group.reason}`
|
|
989
|
-
})
|
|
990
|
-
}
|
|
991
|
-
|
|
992
|
-
function getOmittedCommandCategory (command) {
|
|
993
|
-
const value = `${command.classification || ''} ${command.reason || ''} ${command.command || ''}`.toLowerCase()
|
|
994
|
-
if (/browser|playwright|chromium|firefox|webkit|sauce/.test(value)) {
|
|
995
|
-
return { id: 'browser', label: 'Browser tests', reason: 'require browser or remote-browser setup.' }
|
|
996
|
-
}
|
|
997
|
-
if (/typecheck|typescript compiler|\btsc\b/.test(value)) {
|
|
998
|
-
return { id: 'typecheck', label: 'Typecheck commands', reason: 'do not execute supported runtime tests.' }
|
|
999
|
-
}
|
|
1000
|
-
if (/\bbun\b|\bdeno\b/.test(value)) {
|
|
1001
|
-
return { id: 'unsupported', label: 'Unsupported runtimes', reason: 'are not supported by this validator.' }
|
|
1002
|
-
}
|
|
1003
|
-
if (/pack|build|generated|fixture/.test(value)) {
|
|
1004
|
-
return { id: 'build', label: 'Build-dependent tests', reason: 'require build, package, or fixture setup.' }
|
|
1005
|
-
}
|
|
1006
|
-
if (/service|database|docker|credential/.test(value)) {
|
|
1007
|
-
return { id: 'service', label: 'Service-dependent tests', reason: 'require services or credentials.' }
|
|
1008
|
-
}
|
|
1009
|
-
if (/duplicate|same .*command|already covered/.test(value)) {
|
|
1010
|
-
return { id: 'duplicate', label: 'Duplicate test commands', reason: 'have the same validated runner shape.' }
|
|
1011
|
-
}
|
|
1012
|
-
if (/unsupported|custom runner/.test(value)) {
|
|
1013
|
-
return { id: 'unsupported-runner', label: 'Unsupported test runners', reason: 'cannot be validated live.' }
|
|
1014
|
-
}
|
|
1015
|
-
return { id: 'other', label: 'Other test commands', reason: 'were outside the selected safe validation scope.' }
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
function formatList (values, { format }) {
|
|
1019
|
-
return values.map(value => format(value)).join(', ')
|
|
1020
|
-
}
|
|
1021
|
-
|
|
1022
|
-
function getKeyArtifacts (artifacts) {
|
|
1023
|
-
return [
|
|
1024
|
-
['Markdown report', artifacts.report],
|
|
1025
|
-
['Manifest', artifacts.manifest],
|
|
1026
|
-
['Scenario event artifacts', 'runs/'],
|
|
1027
|
-
['Static diagnosis', artifacts.staticDiagnosis],
|
|
1028
|
-
]
|
|
1029
|
-
}
|
|
1030
|
-
|
|
1031
|
-
function renderConsoleSummary (results, reportPath, runSummary) {
|
|
1032
|
-
const lines = ['', 'Datadog Test Optimization validation summary:']
|
|
1033
|
-
if (runSummary?.runCompleted === true) {
|
|
1034
|
-
lines.push(`Validation completed. Validator exit code: ${runSummary.validatorExitCode}.`)
|
|
1035
|
-
}
|
|
1036
|
-
if (runSummary?.validationCoverage) lines.push(getValidationCoverageSummary(runSummary))
|
|
1037
|
-
const basicReportingResults = getBasicReportingResults(results)
|
|
1038
|
-
const ciWiringResults = getCiWiringResults(results)
|
|
1039
|
-
const advancedFeatureResults = getAdvancedFeatureResults(results)
|
|
1040
|
-
|
|
1041
|
-
lines.push(getConsoleScopeSentence(results))
|
|
1042
|
-
for (const verdict of getFrameworkVerdicts(results)) lines.push(verdict)
|
|
1043
|
-
|
|
1044
|
-
if (basicReportingResults.length > 0) lines.push('Checks:')
|
|
1045
|
-
for (const result of basicReportingResults) {
|
|
1046
|
-
lines.push(formatCompactConsoleResult(result))
|
|
1047
|
-
}
|
|
1048
|
-
|
|
1049
|
-
for (const result of ciWiringResults) {
|
|
1050
|
-
lines.push(formatCompactConsoleResult(result))
|
|
1051
|
-
}
|
|
1052
|
-
for (const result of advancedFeatureResults) {
|
|
1053
|
-
lines.push(formatCompactConsoleResult(result))
|
|
1054
|
-
}
|
|
1055
|
-
|
|
1056
|
-
appendConsoleHowToFix(lines, results)
|
|
1057
|
-
|
|
1058
|
-
lines.push(
|
|
1059
|
-
`Detailed report: ${reportPath}`,
|
|
1060
|
-
`Run artifacts: ${path.dirname(reportPath)}`,
|
|
1061
|
-
`Sharing warning: ${SHARING_WARNING}`,
|
|
1062
|
-
`Evidence warning: ${UNTRUSTED_EVIDENCE_WARNING}`
|
|
1063
|
-
)
|
|
356
|
+
lines.push(`Report: ${reportPath}`)
|
|
1064
357
|
return lines.join('\n')
|
|
1065
358
|
}
|
|
1066
359
|
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
...diagnosticGroups.map(group => `${group.label.toLowerCase()}: ${group.frameworks.join(', ')}`),
|
|
1080
|
-
...omittedGroups,
|
|
1081
|
-
]
|
|
1082
|
-
lines.push('## Scope', '', `Live validation: ${markdownText(live)}.`)
|
|
1083
|
-
if (notValidated.length > 0) lines.push(`Not validated: ${markdownText(notValidated.join('; '))}`)
|
|
1084
|
-
lines.push('')
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
function getConsoleScopeSentence (results) {
|
|
1088
|
-
const live = getUniqueFrameworkLabels(getLiveValidationResults(results))
|
|
1089
|
-
const groups = groupDiagnosticResults(getDiagnosticOnlyResults(results))
|
|
1090
|
-
const excluded = groups.map(group => `${group.label.toLowerCase()}: ${group.frameworks.join(', ')}`)
|
|
1091
|
-
return `Scope: live validation ${live.length > 0 ? live.join(', ') : 'none'}` +
|
|
1092
|
-
`${excluded.length > 0 ? `; not validated ${excluded.join('; ')}` : ''}.`
|
|
1093
|
-
}
|
|
1094
|
-
|
|
1095
|
-
function getUniqueFrameworkLabels (results) {
|
|
1096
|
-
return [...new Set(results.map(getResultFrameworkLabel))]
|
|
1097
|
-
}
|
|
1098
|
-
|
|
1099
|
-
function groupDiagnosticResults (results) {
|
|
1100
|
-
const groups = new Map()
|
|
1101
|
-
for (const result of results) {
|
|
1102
|
-
const category = getDiagnosticCategory(result)
|
|
1103
|
-
const group = groups.get(category.id) || { ...category, frameworks: [] }
|
|
1104
|
-
const label = getResultFrameworkLabel(result)
|
|
1105
|
-
if (!group.frameworks.includes(label)) group.frameworks.push(label)
|
|
1106
|
-
groups.set(category.id, group)
|
|
1107
|
-
}
|
|
1108
|
-
return [...groups.values()]
|
|
1109
|
-
}
|
|
1110
|
-
|
|
1111
|
-
function getDiagnosticCategory (result) {
|
|
1112
|
-
const evidence = result.evidence || {}
|
|
1113
|
-
if (
|
|
1114
|
-
evidence.setupFailed ||
|
|
1115
|
-
evidence.blockedByProjectSetup ||
|
|
1116
|
-
evidence.frameworkStatus === 'requires_manual_setup' ||
|
|
1117
|
-
evidence.frameworkStatus === 'requires_external_service'
|
|
1118
|
-
) {
|
|
1119
|
-
return {
|
|
1120
|
-
id: 'setup',
|
|
1121
|
-
label: 'Requires project setup',
|
|
1122
|
-
reason: 'the required build, service, or fixture setup was not available.',
|
|
1123
|
-
}
|
|
1124
|
-
}
|
|
1125
|
-
if (
|
|
1126
|
-
evidence.frameworkStatus === 'unsupported' ||
|
|
1127
|
-
evidence.frameworkStatus === 'unsupported_by_validator' ||
|
|
1128
|
-
evidence.frameworkStatus === 'detected_not_runnable'
|
|
1129
|
-
) {
|
|
1130
|
-
return {
|
|
1131
|
-
id: 'unsupported',
|
|
1132
|
-
label: 'Unsupported or non-runnable frameworks',
|
|
1133
|
-
reason: 'no supported representative command was available.',
|
|
1134
|
-
}
|
|
1135
|
-
}
|
|
1136
|
-
return {
|
|
1137
|
-
id: 'not-selected',
|
|
1138
|
-
label: 'Not selected for live validation',
|
|
1139
|
-
reason: 'no live Test Optimization conclusion was reached.',
|
|
1140
|
-
}
|
|
360
|
+
/**
|
|
361
|
+
* Returns whether a result lacks a conclusive pass/fail.
|
|
362
|
+
*
|
|
363
|
+
* @param {object|undefined} result validation result
|
|
364
|
+
* @returns {boolean} whether incomplete
|
|
365
|
+
*/
|
|
366
|
+
function isIncomplete (result) {
|
|
367
|
+
return Boolean(result && (
|
|
368
|
+
result.evidence?.validationIncomplete ||
|
|
369
|
+
result.evidence?.manifestIncomplete ||
|
|
370
|
+
['configured_propagation_unverified', 'incomplete'].includes(result.evidence?.conclusion)
|
|
371
|
+
))
|
|
1141
372
|
}
|
|
1142
373
|
|
|
1143
374
|
/**
|
|
1144
|
-
*
|
|
375
|
+
* Returns a display status.
|
|
1145
376
|
*
|
|
1146
|
-
* @param {
|
|
1147
|
-
* @
|
|
1148
|
-
* @returns {void}
|
|
377
|
+
* @param {object} result validation result
|
|
378
|
+
* @returns {string} status
|
|
1149
379
|
*/
|
|
1150
|
-
function
|
|
1151
|
-
|
|
1152
|
-
if (entries.length === 0) return
|
|
1153
|
-
|
|
1154
|
-
lines.push('How to fix:')
|
|
1155
|
-
for (const entry of entries) {
|
|
1156
|
-
lines.push(`${entry.frameworkDisplayName} - ${formatScenarioName(entry.scenario)}:`)
|
|
1157
|
-
for (const recommendation of entry.recommendations) {
|
|
1158
|
-
lines.push(`- ${replaceControlCharacters(sanitizeString(recommendation))}`)
|
|
1159
|
-
}
|
|
1160
|
-
if (entry.ciRemediation?.variants?.length) {
|
|
1161
|
-
for (const variant of entry.ciRemediation.variants) {
|
|
1162
|
-
lines.push(
|
|
1163
|
-
`${variant.name}:`,
|
|
1164
|
-
`Required: ${variant.prerequisite}`,
|
|
1165
|
-
`Required variables: ${variant.requiredValues.map(value => {
|
|
1166
|
-
return `${value.name}${value.source === 'ci-secret-store' ? ' (from CI secret store)' : ''}`
|
|
1167
|
-
}).join(', ')}`,
|
|
1168
|
-
`Recommended variables: ${(variant.recommendedValues || []).map(value => {
|
|
1169
|
-
return `${value.name}=${value.value}`
|
|
1170
|
-
}).join(', ') || 'none'}`,
|
|
1171
|
-
`Optional variables: ${(variant.optionalValues || []).map(value => value.name).join(', ') || 'none'}`,
|
|
1172
|
-
variant.snippet
|
|
1173
|
-
)
|
|
1174
|
-
}
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
380
|
+
function getDisplayStatus (result) {
|
|
381
|
+
return isIncomplete(result) || result.status === 'skip' ? 'INCOMPLETE' : result.status.toUpperCase()
|
|
1177
382
|
}
|
|
1178
383
|
|
|
1179
384
|
/**
|
|
1180
|
-
*
|
|
385
|
+
* Returns results suitable for the top-level table.
|
|
1181
386
|
*
|
|
1182
|
-
* @param {object[]} results
|
|
1183
|
-
* @returns {
|
|
387
|
+
* @param {object[]} results all results
|
|
388
|
+
* @returns {object[]} visible results
|
|
1184
389
|
*/
|
|
1185
|
-
function
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
const recommendations = getResultRecommendations(result)
|
|
1192
|
-
entries.push({
|
|
1193
|
-
frameworkDisplayName: getResultFrameworkLabel(result),
|
|
1194
|
-
scenario: result.scenario,
|
|
1195
|
-
recommendations: recommendations.length > 0 ? recommendations : [getFallbackRecommendation(result)],
|
|
1196
|
-
ciRemediation: isIncompleteResult(result) ? undefined : result.evidence?.ciRemediation,
|
|
390
|
+
function getVisibleResults (results) {
|
|
391
|
+
return results.filter(result => {
|
|
392
|
+
if (result.scenario !== 'all') return true
|
|
393
|
+
return !results.some(candidate => {
|
|
394
|
+
return candidate.frameworkId === result.frameworkId && candidate.scenario === 'basic-reporting'
|
|
1197
395
|
})
|
|
1198
|
-
}
|
|
1199
|
-
|
|
1200
|
-
return entries
|
|
396
|
+
})
|
|
1201
397
|
}
|
|
1202
398
|
|
|
1203
399
|
/**
|
|
1204
|
-
*
|
|
400
|
+
* Returns whether detailed debugging context should be shown.
|
|
1205
401
|
*
|
|
1206
402
|
* @param {object} result validation result
|
|
1207
|
-
* @returns {
|
|
403
|
+
* @returns {boolean} whether details are useful
|
|
1208
404
|
*/
|
|
1209
|
-
function
|
|
1210
|
-
|
|
1211
|
-
const values = [
|
|
1212
|
-
evidence.eventLevelFailure?.recommendation,
|
|
1213
|
-
evidence.localDiagnosis?.recommendation,
|
|
1214
|
-
evidence.commandFailure?.recommendation,
|
|
1215
|
-
evidence.recommendation,
|
|
1216
|
-
...(Array.isArray(evidence.remediation) ? evidence.remediation : []),
|
|
1217
|
-
]
|
|
1218
|
-
|
|
1219
|
-
for (const finding of evidence.monorepoFindings || []) {
|
|
1220
|
-
values.push(finding.recommendation)
|
|
1221
|
-
}
|
|
1222
|
-
|
|
1223
|
-
const seen = new Set()
|
|
1224
|
-
const recommendations = []
|
|
1225
|
-
for (const value of values) {
|
|
1226
|
-
if (typeof value !== 'string' || value.trim() === '' || seen.has(value)) continue
|
|
1227
|
-
seen.add(value)
|
|
1228
|
-
recommendations.push(value)
|
|
1229
|
-
}
|
|
1230
|
-
return recommendations
|
|
405
|
+
function shouldRenderDiagnostics (result) {
|
|
406
|
+
return result.status === 'fail' || result.status === 'error' || isIncomplete(result)
|
|
1231
407
|
}
|
|
1232
408
|
|
|
1233
409
|
/**
|
|
1234
|
-
*
|
|
410
|
+
* Groups results by framework display name.
|
|
1235
411
|
*
|
|
1236
|
-
* @param {object}
|
|
1237
|
-
* @returns {string}
|
|
412
|
+
* @param {object[]} results results
|
|
413
|
+
* @returns {Map<string, object[]>} grouped results
|
|
1238
414
|
*/
|
|
1239
|
-
function
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
return 'Correct or replace the selected CI replay command so it reaches a test result, then rerun CI wiring ' +
|
|
1247
|
-
'validation before changing Datadog configuration.'
|
|
1248
|
-
}
|
|
1249
|
-
return 'Set `NODE_OPTIONS=-r dd-trace/ci/init` and `DD_CIVISIBILITY_AGENTLESS_ENABLED=true` in the identified ' +
|
|
1250
|
-
'CI test step, and provide `DD_API_KEY` from the CI secret store. If a Datadog Agent is available and ' +
|
|
1251
|
-
'reachable by the test process, do not pass `DD_API_KEY` or `DD_CIVISIBILITY_AGENTLESS_ENABLED`.'
|
|
1252
|
-
}
|
|
1253
|
-
if (result.status === 'blocked') {
|
|
1254
|
-
return 'Resolve the blocker described in the report, then rerun validation.'
|
|
415
|
+
function groupByFramework (results) {
|
|
416
|
+
const grouped = new Map()
|
|
417
|
+
for (const result of results) {
|
|
418
|
+
const key = result.frameworkDisplayName
|
|
419
|
+
const values = grouped.get(key) || []
|
|
420
|
+
values.push(result)
|
|
421
|
+
grouped.set(key, values)
|
|
1255
422
|
}
|
|
1256
|
-
return
|
|
1257
|
-
'configuration issue, then rerun this check.'
|
|
423
|
+
return grouped
|
|
1258
424
|
}
|
|
1259
425
|
|
|
1260
426
|
/**
|
|
1261
|
-
*
|
|
427
|
+
* Returns framework labels.
|
|
1262
428
|
*
|
|
1263
|
-
* @param {
|
|
1264
|
-
* @returns {string}
|
|
429
|
+
* @param {object} manifest manifest
|
|
430
|
+
* @returns {Map<string, string>} label map
|
|
1265
431
|
*/
|
|
1266
|
-
function
|
|
1267
|
-
return
|
|
1268
|
-
|
|
1269
|
-
'
|
|
1270
|
-
|
|
1271
|
-
atr: 'Auto Test Retries',
|
|
1272
|
-
'test-management': 'Test Management',
|
|
1273
|
-
all: 'Validation Environment',
|
|
1274
|
-
}[scenario] || scenario
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
function getLiveValidationResults (results) {
|
|
1278
|
-
return results.filter(result => !isDiagnosticOnlyResult(result))
|
|
1279
|
-
}
|
|
1280
|
-
|
|
1281
|
-
function getCiWiringResults (results) {
|
|
1282
|
-
return getLiveValidationResults(results).filter(result => result.scenario === CI_WIRING_SCENARIO)
|
|
1283
|
-
}
|
|
1284
|
-
|
|
1285
|
-
function getBasicReportingResults (results) {
|
|
1286
|
-
return getLiveValidationResults(results).filter(result => result.scenario === 'basic-reporting')
|
|
432
|
+
function getFrameworkLabels (manifest) {
|
|
433
|
+
return new Map((manifest.frameworks || []).map(framework => [
|
|
434
|
+
framework.id,
|
|
435
|
+
`${framework.project?.name || 'project'} (${framework.framework})`,
|
|
436
|
+
]))
|
|
1287
437
|
}
|
|
1288
438
|
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
439
|
+
/**
|
|
440
|
+
* Returns a common artifact directory relative to the report.
|
|
441
|
+
*
|
|
442
|
+
* @param {string[]} artifacts artifact paths
|
|
443
|
+
* @param {string} out report directory
|
|
444
|
+
* @returns {string|undefined} relative directory
|
|
445
|
+
*/
|
|
446
|
+
function getArtifactDirectory (artifacts = [], out) {
|
|
447
|
+
const files = artifacts.filter(value => typeof value === 'string' && path.isAbsolute(value))
|
|
448
|
+
if (files.length === 0) return
|
|
449
|
+
const directory = files.map(filename => path.dirname(filename)).reduce((common, candidate) => {
|
|
450
|
+
while (common !== path.dirname(common) && !isPathInside(common, candidate)) common = path.dirname(common)
|
|
451
|
+
return common
|
|
1292
452
|
})
|
|
453
|
+
return relative(out, directory)
|
|
1293
454
|
}
|
|
1294
455
|
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
'',
|
|
1306
|
-
'| Project | Question | Result | What this means |',
|
|
1307
|
-
'|---|---|---:|---|'
|
|
1308
|
-
)
|
|
1309
|
-
for (const result of liveResults) {
|
|
1310
|
-
lines.push(`| ${markdownText(getResultFrameworkLabel(result))} | ${markdownText(getCheckQuestion(result))} | ` +
|
|
1311
|
-
`${markdownText(getDisplayResultStatus(result))} | ${markdownText(getCompactResultMeaning(result))} |`)
|
|
1312
|
-
}
|
|
1313
|
-
lines.push('')
|
|
1314
|
-
}
|
|
1315
|
-
|
|
1316
|
-
function getScenarioExecutionExplanation (result) {
|
|
1317
|
-
if (result.scenario === 'efd') {
|
|
1318
|
-
return result.status === 'pass'
|
|
1319
|
-
? 'The validator added a temporary passing test, confirmed Datadog detected it as new, and observed the ' +
|
|
1320
|
-
'Early Flake Detection retry evidence.'
|
|
1321
|
-
: 'The validator added a temporary passing test and checked whether Datadog detected and retried it as new.'
|
|
1322
|
-
}
|
|
1323
|
-
if (result.scenario === 'atr') {
|
|
1324
|
-
return result.status === 'pass'
|
|
1325
|
-
? 'The validator added a temporary test that fails once, then observed Datadog retry it and the retry pass.'
|
|
1326
|
-
: 'The validator added a temporary fail-once test and checked whether Datadog retried it.'
|
|
1327
|
-
}
|
|
1328
|
-
if (result.scenario === 'test-management') {
|
|
1329
|
-
return result.status === 'pass'
|
|
1330
|
-
? 'The validator added a temporary target test, matched it through Test Management, and observed the ' +
|
|
1331
|
-
'quarantine tag.'
|
|
1332
|
-
: 'The validator added a temporary target test and checked whether Test Management matched and tagged it.'
|
|
1333
|
-
}
|
|
1334
|
-
}
|
|
1335
|
-
|
|
1336
|
-
function getFrameworkVerdicts (results) {
|
|
1337
|
-
const liveResults = getLiveValidationResults(results)
|
|
1338
|
-
const frameworkResults = new Map()
|
|
1339
|
-
for (const result of liveResults) {
|
|
1340
|
-
const entries = frameworkResults.get(result.frameworkId) || []
|
|
1341
|
-
entries.push(result)
|
|
1342
|
-
frameworkResults.set(result.frameworkId, entries)
|
|
1343
|
-
}
|
|
1344
|
-
|
|
1345
|
-
const verdicts = []
|
|
1346
|
-
for (const entries of frameworkResults.values()) {
|
|
1347
|
-
const label = getResultFrameworkLabel(entries[0])
|
|
1348
|
-
const basic = entries.find(result => result.scenario === 'basic-reporting')
|
|
1349
|
-
const ciWiring = entries.find(result => result.scenario === CI_WIRING_SCENARIO)
|
|
1350
|
-
if (basic?.status === 'pass' && ciWiring?.status === 'fail') {
|
|
1351
|
-
verdicts.push(`${label}: dd-trace successfully reports this test suite, but the selected CI job does not ` +
|
|
1352
|
-
'load dd-trace when it runs the tests.')
|
|
1353
|
-
} else if (basic?.status === 'pass' && isIncompleteResult(ciWiring)) {
|
|
1354
|
-
verdicts.push(`${label}: dd-trace successfully reports this test suite, but the selected CI command did ` +
|
|
1355
|
-
'not reach a test result. No live CI-wiring conclusion was reached.')
|
|
1356
|
-
} else if (basic?.status === 'pass' && ciWiring?.status === 'pass') {
|
|
1357
|
-
verdicts.push(`${label}: this test suite reports successfully, including from the selected CI job.`)
|
|
1358
|
-
} else if (basic && basic.status !== 'pass') {
|
|
1359
|
-
verdicts.push(`${label}: the selected tests did not report successfully, so no CI wiring conclusion was ` +
|
|
1360
|
-
'reached.')
|
|
1361
|
-
} else if (basic?.status === 'pass') {
|
|
1362
|
-
verdicts.push(`${label}: this test suite reports successfully when dd-trace is initialized.`)
|
|
1363
|
-
}
|
|
1364
|
-
}
|
|
1365
|
-
if (liveResults.length === 0) {
|
|
1366
|
-
verdicts.push('No live Test Optimization validation ran. This result is incomplete; no Basic Reporting, CI ' +
|
|
1367
|
-
'wiring, or advanced-feature conclusion was reached.')
|
|
1368
|
-
}
|
|
1369
|
-
return verdicts
|
|
456
|
+
/**
|
|
457
|
+
* Checks path containment.
|
|
458
|
+
*
|
|
459
|
+
* @param {string} root root path
|
|
460
|
+
* @param {string} filename candidate path
|
|
461
|
+
* @returns {boolean} containment
|
|
462
|
+
*/
|
|
463
|
+
function isPathInside (root, filename) {
|
|
464
|
+
const value = path.relative(root, filename)
|
|
465
|
+
return value === '' || (!value.startsWith('..') && !path.isAbsolute(value))
|
|
1370
466
|
}
|
|
1371
467
|
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
}[result.scenario] || formatScenarioName(result.scenario)
|
|
468
|
+
/**
|
|
469
|
+
* Formats an execution status.
|
|
470
|
+
*
|
|
471
|
+
* @param {string|undefined} status status id
|
|
472
|
+
* @returns {string} display status
|
|
473
|
+
*/
|
|
474
|
+
function formatExecutionStatus (status) {
|
|
475
|
+
return String(status || 'incomplete').replaceAll('_', ' ').toUpperCase()
|
|
1381
476
|
}
|
|
1382
477
|
|
|
1383
|
-
function
|
|
1384
|
-
if (
|
|
1385
|
-
|
|
1386
|
-
return
|
|
1387
|
-
}
|
|
1388
|
-
if (result.scenario === CI_WIRING_SCENARIO && result.status === 'fail') {
|
|
1389
|
-
if (result.evidence?.nodeOptionsRemoval) {
|
|
1390
|
-
return 'CI ran tests, but a package script removed the dd-trace preload before the test runner started.'
|
|
1391
|
-
}
|
|
1392
|
-
if (result.evidence?.lateInitialization?.length > 0) {
|
|
1393
|
-
return 'CI initializes dd-trace after the test runner starts, so no test data was reported.'
|
|
1394
|
-
}
|
|
1395
|
-
return 'CI ran tests without initializing dd-trace, so no test data was reported.'
|
|
478
|
+
function formatCleanupStatus (cleanup) {
|
|
479
|
+
if (cleanup?.status === 'completed') {
|
|
480
|
+
const removed = (cleanup.filesRemoved || 0) + (cleanup.directoriesRemoved || 0)
|
|
481
|
+
return `completed${removed > 0 ? ` (${removed} temporary path${removed === 1 ? '' : 's'} removed)` : ''}`
|
|
1396
482
|
}
|
|
1397
|
-
if (
|
|
1398
|
-
|
|
483
|
+
if (cleanup?.status === 'retained_by_request') return 'temporary files retained by approved request'
|
|
484
|
+
if (cleanup?.status === 'incomplete') {
|
|
485
|
+
const retained = (cleanup.filesRetained || 0) + (cleanup.directoriesRetained || 0)
|
|
486
|
+
return `incomplete${retained > 0 ? ` (${retained} temporary path${retained === 1 ? '' : 's'} retained)` : ''}`
|
|
1399
487
|
}
|
|
1400
|
-
|
|
1401
|
-
if (explanation) return explanation
|
|
1402
|
-
return result.diagnosis
|
|
1403
|
-
}
|
|
1404
|
-
|
|
1405
|
-
function formatCompactConsoleResult (result) {
|
|
1406
|
-
return `${getDisplayResultStatus(result)} ${getResultFrameworkLabel(result)} - ${getCheckQuestion(result)} ` +
|
|
1407
|
-
`- ${replaceControlCharacters(sanitizeString(getCompactResultMeaning(result)))}`
|
|
1408
|
-
}
|
|
1409
|
-
|
|
1410
|
-
function isIncompleteResult (result) {
|
|
1411
|
-
return result?.evidence?.manifestIncomplete === true || result?.evidence?.validationIncomplete === true
|
|
1412
|
-
}
|
|
1413
|
-
|
|
1414
|
-
function getDisplayResultStatus (result) {
|
|
1415
|
-
if (result.evidence?.validationNotSelected === true) return 'NOT CHECKED'
|
|
1416
|
-
return isIncompleteResult(result) ? 'INCOMPLETE' : result.status.toUpperCase()
|
|
488
|
+
return 'not completed'
|
|
1417
489
|
}
|
|
1418
490
|
|
|
1419
491
|
/**
|
|
1420
|
-
*
|
|
492
|
+
* Returns a display scenario name.
|
|
1421
493
|
*
|
|
1422
|
-
* @param {
|
|
1423
|
-
* @
|
|
1424
|
-
* @param {object[]} input.results sanitized validation results
|
|
1425
|
-
* @param {object} input.runSummary run metadata
|
|
1426
|
-
* @returns {object[]} results including unselected check rows
|
|
494
|
+
* @param {string} scenario scenario id
|
|
495
|
+
* @returns {string} display name
|
|
1427
496
|
*/
|
|
1428
|
-
function
|
|
1429
|
-
|
|
1430
|
-
if (omittedScenarios.length === 0) return results
|
|
1431
|
-
|
|
1432
|
-
const selected = formatScenarioList(runSummary.checkedScenarios || [])
|
|
1433
|
-
const additions = []
|
|
1434
|
-
for (const framework of manifest.frameworks || []) {
|
|
1435
|
-
if (framework.status !== 'runnable') continue
|
|
1436
|
-
for (const scenario of omittedScenarios) {
|
|
1437
|
-
if (results.some(result => result.frameworkId === framework.id && result.scenario === scenario)) continue
|
|
1438
|
-
additions.push({
|
|
1439
|
-
frameworkId: framework.id,
|
|
1440
|
-
scenario,
|
|
1441
|
-
status: 'skip',
|
|
1442
|
-
diagnosis: `Not checked because this run was limited to ${selected}.`,
|
|
1443
|
-
evidence: { validationNotSelected: true },
|
|
1444
|
-
artifacts: [],
|
|
1445
|
-
})
|
|
1446
|
-
}
|
|
1447
|
-
}
|
|
1448
|
-
return [...results, ...additions]
|
|
497
|
+
function getScenarioName (scenario) {
|
|
498
|
+
return SCENARIO_NAMES[scenario] || scenario
|
|
1449
499
|
}
|
|
1450
500
|
|
|
1451
501
|
/**
|
|
1452
|
-
* Formats
|
|
502
|
+
* Formats a Markdown table cell.
|
|
1453
503
|
*
|
|
1454
|
-
* @param {
|
|
1455
|
-
* @returns {string}
|
|
504
|
+
* @param {unknown} value cell value
|
|
505
|
+
* @returns {string} escaped text
|
|
1456
506
|
*/
|
|
1457
|
-
function
|
|
1458
|
-
|
|
1459
|
-
return 'Validation coverage: complete. All checks selected by the full workflow produced a result.'
|
|
1460
|
-
}
|
|
1461
|
-
const checked = formatScenarioList(runSummary.checkedScenarios || [])
|
|
1462
|
-
const omitted = formatScenarioList(runSummary.omittedScenarios || [])
|
|
1463
|
-
return `Validation coverage: partial. Checked ${checked || 'no checks'}; ` +
|
|
1464
|
-
`${omitted ? `did not check ${omitted}` : 'one or more selected checks were incomplete'}.`
|
|
507
|
+
function cell (value) {
|
|
508
|
+
return plain(value).replaceAll('|', String.raw`\|`)
|
|
1465
509
|
}
|
|
1466
510
|
|
|
1467
511
|
/**
|
|
1468
|
-
* Formats
|
|
512
|
+
* Formats inline code.
|
|
1469
513
|
*
|
|
1470
|
-
* @param {
|
|
1471
|
-
* @returns {string}
|
|
514
|
+
* @param {unknown} value code value
|
|
515
|
+
* @returns {string} Markdown code
|
|
1472
516
|
*/
|
|
1473
|
-
function
|
|
1474
|
-
return
|
|
1475
|
-
}
|
|
1476
|
-
|
|
1477
|
-
function getFrameworkLabels (manifest) {
|
|
1478
|
-
const labels = new Map()
|
|
1479
|
-
for (const framework of manifest.frameworks || []) {
|
|
1480
|
-
labels.set(framework.id, getFrameworkLabel(framework))
|
|
1481
|
-
}
|
|
1482
|
-
return labels
|
|
1483
|
-
}
|
|
1484
|
-
|
|
1485
|
-
function getFrameworkLabel (framework) {
|
|
1486
|
-
const projectName = framework.project?.name
|
|
1487
|
-
const frameworkName = formatFrameworkName(framework.framework)
|
|
1488
|
-
if (!projectName) return framework.id
|
|
1489
|
-
return `${projectName} (${frameworkName})`
|
|
1490
|
-
}
|
|
1491
|
-
|
|
1492
|
-
function getResultFrameworkLabel (result) {
|
|
1493
|
-
return result.frameworkDisplayName || result.frameworkId
|
|
517
|
+
function code (value) {
|
|
518
|
+
return `\`${plain(value).replaceAll('`', String.raw`\u0060`)}\``
|
|
1494
519
|
}
|
|
1495
520
|
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
521
|
+
/**
|
|
522
|
+
* Sanitizes one line of report text.
|
|
523
|
+
*
|
|
524
|
+
* @param {unknown} value text
|
|
525
|
+
* @returns {string} safe text
|
|
526
|
+
*/
|
|
527
|
+
function plain (value) {
|
|
528
|
+
return sanitizeString(String(value ?? '')).replaceAll(/\p{Cc}+/gu, ' ').trim()
|
|
1499
529
|
}
|
|
1500
530
|
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
531
|
+
/**
|
|
532
|
+
* Returns a relative artifact path.
|
|
533
|
+
*
|
|
534
|
+
* @param {string} root base directory
|
|
535
|
+
* @param {string} filename file path
|
|
536
|
+
* @returns {string} relative path
|
|
537
|
+
*/
|
|
538
|
+
function relative (root, filename) {
|
|
539
|
+
return path.relative(root, filename) || '.'
|
|
1506
540
|
}
|
|
1507
541
|
|
|
1508
542
|
module.exports = { writePendingReport, writeReport }
|