dd-trace 5.116.0 → 5.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/ci/diagnose.js +162 -21
  2. package/ci/init.js +0 -1
  3. package/ci/runbook.md +187 -180
  4. package/ci/test-optimization-validation/approval-artifacts.js +3 -1
  5. package/ci/test-optimization-validation/approval.js +52 -15
  6. package/ci/test-optimization-validation/ci-command-candidate.js +26 -32
  7. package/ci/test-optimization-validation/ci-discovery.js +1 -1
  8. package/ci/test-optimization-validation/ci-package-scripts.js +141 -0
  9. package/ci/test-optimization-validation/ci-remediation.js +112 -39
  10. package/ci/test-optimization-validation/cli.js +516 -681
  11. package/ci/test-optimization-validation/command-blocker.js +188 -33
  12. package/ci/test-optimization-validation/command-output-policy.js +3 -26
  13. package/ci/test-optimization-validation/command-runner.js +189 -245
  14. package/ci/test-optimization-validation/environment.js +90 -0
  15. package/ci/test-optimization-validation/executable.js +159 -388
  16. package/ci/test-optimization-validation/framework-adapters/cucumber.js +119 -0
  17. package/ci/test-optimization-validation/framework-adapters/cypress.js +107 -0
  18. package/ci/test-optimization-validation/framework-adapters/playwright.js +181 -0
  19. package/ci/test-optimization-validation/generated-files.js +75 -13
  20. package/ci/test-optimization-validation/generated-test-contract.js +283 -0
  21. package/ci/test-optimization-validation/generated-verifier.js +49 -16
  22. package/ci/test-optimization-validation/literal-environment.js +57 -0
  23. package/ci/test-optimization-validation/manifest-loader.js +2 -22
  24. package/ci/test-optimization-validation/manifest-scaffold.js +816 -515
  25. package/ci/test-optimization-validation/manifest-schema.js +494 -702
  26. package/ci/test-optimization-validation/offline-fixtures.js +4 -1
  27. package/ci/test-optimization-validation/plan-writer.js +243 -984
  28. package/ci/test-optimization-validation/preflight-runner.js +49 -55
  29. package/ci/test-optimization-validation/redaction.js +2 -1
  30. package/ci/test-optimization-validation/report-writer.js +391 -1357
  31. package/ci/test-optimization-validation/result-semantics.js +86 -0
  32. package/ci/test-optimization-validation/runner-command.js +249 -0
  33. package/ci/test-optimization-validation/runner-contract.js +664 -0
  34. package/ci/test-optimization-validation/scenarios/auto-test-retries.js +29 -3
  35. package/ci/test-optimization-validation/scenarios/basic-reporting.js +240 -543
  36. package/ci/test-optimization-validation/scenarios/ci-wiring.js +450 -670
  37. package/ci/test-optimization-validation/scenarios/early-flake-detection.js +4 -3
  38. package/ci/test-optimization-validation/scenarios/helpers.js +67 -9
  39. package/ci/test-optimization-validation/scenarios/test-management.js +4 -3
  40. package/ci/test-optimization-validation/source-text.js +87 -0
  41. package/ci/test-optimization-validation/test-output.js +12 -22
  42. package/ext/tags.d.ts +1 -0
  43. package/index.d.ts +7 -0
  44. package/package.json +3 -3
  45. package/packages/datadog-instrumentations/src/cucumber.js +11 -12
  46. package/packages/datadog-instrumentations/src/helpers/channel.js +74 -0
  47. package/packages/datadog-instrumentations/src/helpers/hooks.js +2 -0
  48. package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js +1 -0
  49. package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/openai-agents.js +31 -0
  50. package/packages/datadog-instrumentations/src/jest.js +18 -17
  51. package/packages/datadog-instrumentations/src/mocha/main.js +79 -30
  52. package/packages/datadog-instrumentations/src/openai-agents.js +159 -0
  53. package/packages/datadog-instrumentations/src/playwright.js +3 -24
  54. package/packages/datadog-instrumentations/src/vitest-main-no-worker-init.js +1 -1
  55. package/packages/datadog-instrumentations/src/vitest-main.js +14 -26
  56. package/packages/datadog-instrumentations/src/vitest-util.js +0 -7
  57. package/packages/datadog-instrumentations/src/vitest-worker.js +5 -8
  58. package/packages/datadog-plugin-cucumber/src/index.js +2 -1
  59. package/packages/datadog-plugin-mocha/src/index.js +2 -1
  60. package/packages/datadog-plugin-openai-agents/src/index.js +74 -0
  61. package/packages/datadog-plugin-openai-agents/src/integration.js +503 -0
  62. package/packages/datadog-plugin-openai-agents/src/processor.js +108 -0
  63. package/packages/datadog-plugin-openai-agents/src/util.js +60 -0
  64. package/packages/datadog-plugin-playwright/src/index.js +4 -1
  65. package/packages/datadog-plugin-vitest/src/index.js +5 -5
  66. package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +48 -49
  67. package/packages/dd-trace/src/ci-visibility/requests/get-library-configuration.js +161 -33
  68. package/packages/dd-trace/src/ci-visibility/test-optimization-http-cache-schema.js +6 -4
  69. package/packages/dd-trace/src/config/generated-config-types.d.ts +2 -0
  70. package/packages/dd-trace/src/config/supported-configurations.json +7 -0
  71. package/packages/dd-trace/src/encode/span-stats.js +7 -1
  72. package/packages/dd-trace/src/exporter.js +16 -9
  73. package/packages/dd-trace/src/llmobs/index.js +9 -1
  74. package/packages/dd-trace/src/llmobs/plugins/ai/util.js +104 -22
  75. package/packages/dd-trace/src/llmobs/plugins/openai/index.js +5 -9
  76. package/packages/dd-trace/src/llmobs/plugins/openai/utils.js +20 -0
  77. package/packages/dd-trace/src/llmobs/plugins/openai-agents/utils.js +321 -0
  78. package/packages/dd-trace/src/llmobs/sdk.js +2 -1
  79. package/packages/dd-trace/src/llmobs/span_processor.js +7 -2
  80. package/packages/dd-trace/src/llmobs/tagger.js +23 -3
  81. package/packages/dd-trace/src/llmobs/util.js +56 -3
  82. package/packages/dd-trace/src/openfeature/agentless_configuration_source.js +3 -6
  83. package/packages/dd-trace/src/openfeature/configuration_source.js +5 -8
  84. package/packages/dd-trace/src/opentelemetry/metrics/otlp_span_stats_transformer.js +5 -1
  85. package/packages/dd-trace/src/plugins/ci_plugin.js +9 -3
  86. package/packages/dd-trace/src/plugins/index.js +1 -0
  87. package/packages/dd-trace/src/plugins/outbound.js +4 -1
  88. package/packages/dd-trace/src/profiling/profilers/space.js +1 -1
  89. package/packages/dd-trace/src/profiling/profilers/wall.js +1 -0
  90. package/packages/dd-trace/src/span_processor.js +1 -1
  91. package/packages/dd-trace/src/span_stats.js +22 -4
  92. package/ci/test-optimization-validation/command-suitability.js +0 -471
  93. package/ci/test-optimization-validation/init-probe-preload.js +0 -163
  94. package/ci/test-optimization-validation/init-probe.js +0 -246
  95. package/ci/test-optimization-validation/late-initialization.js +0 -63
  96. package/ci/test-optimization-validation/local-command.js +0 -163
  97. package/ci/test-optimization-validation/setup-runner.js +0 -97
  98. package/ci/test-optimization-validation-manifest.schema.json +0 -1
@@ -1,79 +1,57 @@
1
1
  'use strict'
2
2
 
3
3
  const crypto = require('node:crypto')
4
- const fs = require('node:fs')
5
4
  const path = require('node:path')
6
5
 
7
- const { getArtifactId } = require('./artifact-id')
6
+ const { VERSION } = require('../../version')
8
7
  const { writeApprovalArtifacts } = require('./approval-artifacts')
9
- const { getCommandOutputPaths } = require('./command-output-policy')
10
- const { getCommandSuitabilityError } = require('./command-suitability')
11
8
  const { serializeApprovalCommand } = require('./command-runner')
9
+ const { getUnavailableExecutable } = require('./executable')
10
+ const { sanitizeString } = require('./redaction')
12
11
  const {
13
- getApprovedExecutable,
14
- getUnavailableExecutable,
15
- } = require('./executable')
16
- const { getCiWiringCommand, getDatadogCleanCommand, getLocalValidationCommand } = require('./local-command')
17
- const {
18
- getOfflineFixturePaths,
19
- getOfflineScenarioNames,
20
- } = require('./offline-fixtures')
21
- const { sanitizeEnv, sanitizeString } = require('./redaction')
22
- const { getBasicReportingCommand } = require('./scenarios/basic-reporting')
12
+ getBasicCommand,
13
+ getGeneratedCommand,
14
+ } = require('./runner-command')
23
15
  const { writeFileSafely } = require('./safe-files')
24
16
 
25
17
  const VALIDATOR_PATH = path.resolve(__dirname, '..', 'validate-test-optimization.js')
26
- const APPROVAL_SUMMARY_FILENAME = 'approval-summary.md'
27
18
  const EXECUTION_PLAN_FILENAME = 'execution-plan.md'
28
- const GENERATED_SCENARIO_DETAILS = {
29
- 'basic-pass': {
30
- heading: 'Advanced Check: Early Flake Detection',
31
- description: 'Creates a temporary passing test, records it with Early Flake Detection disabled, then enables ' +
32
- 'the feature and checks that Datadog recognizes the test as new and retries it.',
33
- },
34
- 'atr-fail-once': {
35
- heading: 'Advanced Check: Auto Test Retries',
36
- description: 'Creates a temporary test that fails on its first attempt, then checks that Datadog retries it ' +
37
- 'and observes the passing attempt.',
38
- },
39
- 'test-management-target': {
40
- heading: 'Advanced Check: Test Management',
41
- description: 'Creates a temporary target test, supplies a quarantine setting for that test, then checks that ' +
42
- 'Datadog applies the setting.',
43
- },
44
- }
45
- const FRAMEWORK_NAMES = {
46
- jest: 'Jest',
47
- karma: 'Karma',
48
- mocha: 'Mocha',
49
- 'node:test': 'Node.js test runner',
50
- playwright: 'Playwright',
51
- vitest: 'Vitest',
19
+ const SCENARIO_TO_GENERATED_ID = {
20
+ atr: 'atr-fail-once',
21
+ efd: 'basic-pass',
22
+ 'test-management': 'test-management-target',
52
23
  }
53
- // eslint-disable-next-line prefer-regex-literals
54
- const CONTROL_CHARACTERS_PATTERN = new RegExp(String.raw`[\u0000-\u001F\u007F]+`, 'g')
55
24
 
56
25
  /**
57
26
  * Produces the deterministic execution plan shown before live validation.
58
27
  *
59
28
  * @param {object} input plan inputs
60
- * @param {object} input.manifest normalized validation manifest
61
- * @param {string} input.out validation output directory
62
- * @param {string[]} [input.selectedFrameworkIds] explicitly selected framework entries
63
- * @param {string|null} [input.requestedScenario] explicitly selected scenario
64
- * @param {boolean} [input.keepTempFiles] whether generated files should be retained
65
- * @param {boolean} [input.verbose] whether command progress should be printed
66
29
  * @returns {string} Markdown execution plan
67
30
  */
68
- function formatExecutionPlan ({
31
+ function formatExecutionPlan (input) {
32
+ return formatExecutionPlanArtifacts(input).plan
33
+ }
34
+
35
+ /**
36
+ * Writes approval artifacts and the customer-visible plan without running project code.
37
+ *
38
+ * @param {object} input plan inputs
39
+ * @param {object} input.manifest normalized manifest
40
+ * @param {string} input.out output directory
41
+ * @param {string[]} [input.selectedFrameworkIds] selected framework ids
42
+ * @param {string|null} [input.requestedScenario] selected scenario
43
+ * @param {boolean} [input.keepTempFiles] retain temporary files
44
+ * @param {boolean} [input.verbose] print progress
45
+ * @returns {{plan: string}} written plan
46
+ */
47
+ function formatExecutionPlanArtifacts ({
69
48
  manifest,
70
49
  out,
71
50
  selectedFrameworkIds = [],
72
- requestedScenario,
51
+ requestedScenario = null,
73
52
  keepTempFiles = false,
74
53
  verbose = false,
75
54
  }) {
76
- assertPlannedExecutablesAvailable(manifest, requestedScenario)
77
55
  const offlineFixtureNonce = crypto.randomBytes(16).toString('hex')
78
56
  const approvalArtifacts = writeApprovalArtifacts({
79
57
  manifest,
@@ -84,1037 +62,318 @@ function formatExecutionPlan ({
84
62
  keepTempFiles,
85
63
  verbose,
86
64
  })
87
- const approvalDigest = approvalArtifacts.digest
88
- const validatorArgv = getValidatorArgv({
89
- approvedPlanSha256: approvalDigest,
90
- approvalJsonPath: approvalArtifacts.approvalJsonPath,
91
- repositoryRoot: manifest.repository.root,
92
- })
93
- const coveredFileVerification = process.platform === 'win32'
94
- ? []
95
- : [
96
- 'Optional: verify every listed dd-trace package and command executable file against its recorded SHA-256:',
97
- '',
98
- codeBlock(sanitizeString(serializeApprovalCommand({
99
- argv: ['shasum', '-a', '256', '--quiet', '-c', approvalArtifacts.coveredFilesPath],
100
- cwd: manifest.repository.root,
101
- usesShell: false,
102
- }))),
103
- '',
104
- ]
105
-
106
- const lines = [
107
- '# Test Optimization Validation Execution Plan',
108
- '',
109
- `Repository: ${inlineCode(manifest.repository.root)}`,
110
- `Manifest: ${inlineCode(manifest.__path)}`,
111
- `Results: ${inlineCode(out)}`,
112
- '',
113
- '## What Will Be Validated',
114
- '',
115
- 'The validator runs selected project tests without Datadog to confirm they work normally, then runs the same ' +
116
- 'tests with Datadog initialized to check that test data is reported. When a CI test command can be replayed, ' +
117
- 'it also checks whether the configuration from that CI job reaches the test process. Temporary tests are ' +
118
- 'used for the advanced feature checks.',
119
- '',
65
+ const validatorArgv = [
66
+ process.execPath,
67
+ VALIDATOR_PATH,
68
+ '--run-approved-plan',
69
+ approvalArtifacts.approvalJsonPath,
70
+ '--sha256',
71
+ approvalArtifacts.digest,
120
72
  ]
121
-
122
- for (const framework of manifest.frameworks) {
123
- const label = formatFrameworkLabel(framework, manifest.repository.root)
124
- lines.push(`- **${plainText(label)}**: ${formatFrameworkStatus(framework.status)}`)
125
- if (framework.status !== 'runnable') {
126
- for (const note of framework.notes || []) lines.push(` - ${plainText(note)}`)
127
- }
128
- }
129
-
130
- lines.push(
131
- '',
132
- '## Test Commands',
133
- '',
134
- 'Environment values that affect a project command are shown inline after secret-like values are replaced with ' +
135
- '`<redacted>`. Repository files in a command are shown relative to its stated working directory. Datadog ' +
136
- 'preloads are shown as package names; the validator resolves them from the installed `dd-trace` package.',
137
- ''
138
- )
139
- for (const framework of manifest.frameworks) {
140
- if (framework.status !== 'runnable') continue
141
-
142
- appendFrameworkExecutions(
143
- lines,
144
- framework,
145
- requestedScenario,
146
- manifest.repository.root,
147
- out,
148
- offlineFixtureNonce
149
- )
150
- }
151
- appendCommandIntegrity(lines, manifest, requestedScenario)
152
-
153
- lines.push(
154
- '',
155
- '## Start the Validation',
156
- '',
157
- '`validate-test-optimization.js` is the local validator included with the installed `dd-trace` package. ' +
158
- 'After approval, it creates bounded filesystem cache fixtures, performs every check listed above, writes ' +
159
- 'events to local artifacts, and removes temporary fixtures and tests afterward. It does not open a listener ' +
160
- 'or use a network endpoint.',
161
- '',
162
- 'The validator wrote the exact approval material to these local files without running project code:',
163
- '',
164
- `- Approval details: ${inlineCode(getRepositoryRelativePath(
165
- manifest.repository.root,
166
- approvalArtifacts.approvalJsonPath
167
- ))}`,
168
- `- Covered file checksums: ${inlineCode(getRepositoryRelativePath(
169
- manifest.repository.root,
170
- approvalArtifacts.coveredFilesPath
171
- ))}`,
172
- '',
173
- 'The JSON contains the sanitized command shapes, generated test source, selected options, file fingerprints, ' +
174
- 'and executable identities covered by approval. It is an internal diagnostic artifact and may contain ' +
175
- 'repository paths or CI metadata.',
176
- '',
177
- 'Optional: independently hash the approval JSON with a standard system tool:',
178
- '',
179
- codeBlock(formatIndependentHashCommand(approvalArtifacts.approvalJsonPath)),
180
- '',
181
- `Expected SHA-256: ${inlineCode(approvalDigest)}`,
182
- '',
183
- ...coveredFileVerification,
184
- 'Immediately before project code runs, the validator verifies the saved approval JSON against the SHA-256 in ' +
185
- 'the command below, then reconstructs the approval material from the current manifest, validator package, ' +
186
- 'generated tests, and executables. Both checks must match. This detects changes after review; it does not ' +
187
- 'verify where the installed `dd-trace` package came from.',
188
- '',
189
- 'Run the approved validation command:',
190
- '',
191
- codeBlock(sanitizeString(serializeApprovalCommand({
192
- argv: validatorArgv,
193
- cwd: manifest.repository.root,
194
- usesShell: false,
195
- }))),
196
- '',
197
- `Working directory: ${inlineCode(manifest.repository.root)}`,
198
- '',
199
- 'The validator supplies diagnostic Datadog settings from cache files only while these local checks run; those ' +
200
- 'settings are not customer CI recommendations. dd-trace makes no network requests in this validation mode. ' +
201
- 'A setup or test command may still use the network unless the execution sandbox blocks it.',
202
- '',
203
- 'These checks run the project commands listed above. The validator does not require real Datadog ' +
204
- 'credentials, inspect credential stores, or upload validation results. Project tests are arbitrary code and ' +
205
- 'can forge diagnostic cache or event data, so this result is diagnostic evidence, not a security attestation. ' +
206
- 'Review the exact commands before approving them for this environment.'
207
- )
208
-
209
- const plan = lines.join('\n')
210
- const approvalSummary = formatApprovalSummary({
73
+ const plan = formatApprovalPlan({
211
74
  approvalArtifacts,
212
- approvalDigest,
213
75
  manifest,
214
76
  out,
215
77
  requestedScenario,
216
78
  validatorArgv,
217
79
  })
218
80
  writeFileSafely(out, getExecutionPlanPath(out), `${plan}\n`, 'validation execution plan')
219
- writeFileSafely(out, getApprovalSummaryPath(out), `${approvalSummary}\n`, 'validation approval summary')
220
- return plan
81
+ return { plan }
221
82
  }
222
83
 
223
84
  /**
224
- * Returns the bounded customer-facing summary an agent presents before approval.
85
+ * Formats the complete bounded approval plan.
225
86
  *
226
- * @param {object} input summary inputs
227
- * @param {object} input.approvalArtifacts written approval artifact paths
228
- * @param {string} input.approvalDigest approval material digest
229
- * @param {object} input.manifest normalized validation manifest
230
- * @param {string} input.out validation output directory
231
- * @param {string|null|undefined} input.requestedScenario selected scenario
232
- * @param {string[]} input.validatorArgv approved validator command
233
- * @returns {string} Markdown approval summary
87
+ * @param {object} input formatting inputs
88
+ * @param {object} input.approvalArtifacts approval artifact paths and digest
89
+ * @param {object} input.manifest normalized manifest
90
+ * @param {string} input.out output directory
91
+ * @param {string|null} input.requestedScenario selected scenario
92
+ * @param {string[]} input.validatorArgv exact validator command
93
+ * @returns {string} Markdown plan
234
94
  */
235
- function formatApprovalSummary ({
236
- approvalArtifacts,
237
- approvalDigest,
238
- manifest,
239
- out,
240
- requestedScenario,
241
- validatorArgv,
242
- }) {
243
- const repositoryRoot = manifest.repository.root
244
- const coveredFileVerification = process.platform === 'win32'
245
- ? []
246
- : [
247
- 'Optional: verify every covered manifest, validator, and executable file:',
248
- '',
249
- codeBlock(sanitizeString(serializeApprovalCommand({
250
- argv: ['shasum', '-a', '256', '--quiet', '-c', approvalArtifacts.coveredFilesPath],
251
- cwd: repositoryRoot,
252
- usesShell: false,
253
- }))),
254
- '',
255
- ]
95
+ function formatApprovalPlan ({ approvalArtifacts, manifest, out, requestedScenario, validatorArgv }) {
96
+ const root = manifest.repository.root
256
97
  const lines = [
257
- '# Test Optimization Validation Approval Summary',
98
+ '# Test Optimization Validation Plan',
258
99
  '',
259
- `Repository: ${inlineCode(repositoryRoot)}`,
260
- `Detailed execution plan: ${inlineCode(getRepositoryRelativePath(repositoryRoot, getExecutionPlanPath(out)))}`,
100
+ `Repository: ${inline(root)}`,
101
+ `Validator: ${inline(`dd-trace ${VERSION}`)}`,
102
+ `Results: ${inline(relative(root, out))}`,
261
103
  '',
262
- 'This summary shows every project command and the exact temporary test source. The detailed plan contains ' +
263
- 'the offline-fixture, artifact, executable-integrity, and checksum details covered by the same approval hash.',
104
+ 'The validator will run only the displayed `node <repository-contained-runner> <one-test-file>` commands. The ' +
105
+ 'runner may be an exact repository-owned Node test wrapper. The validator itself will not directly invoke ' +
106
+ 'package managers, shells, setup commands, or CI commands; approved repository code may start subprocesses.',
264
107
  '',
265
- '## Scope',
108
+ '## Checks',
266
109
  '',
267
110
  ]
268
111
 
269
112
  for (const framework of manifest.frameworks) {
270
- const label = formatFrameworkLabel(framework, repositoryRoot)
271
- lines.push(`- **${plainText(label)}**: ${formatFrameworkStatus(framework.status)}`)
272
- if (framework.status !== 'runnable' && framework.notes?.[0]) {
273
- lines.push(` - ${plainText(framework.notes[0])}`)
274
- }
275
- }
276
-
277
- lines.push('', '## Commands', '')
278
- for (const framework of manifest.frameworks) {
279
- if (framework.status !== 'runnable') continue
280
-
281
- appendApprovalSummaryFramework(lines, framework, requestedScenario, repositoryRoot)
282
- }
283
-
284
- lines.push(
285
- '## Safety and Outputs',
286
- '',
287
- `- Local results: ${inlineCode(getRepositoryRelativePath(repositoryRoot, out))}`,
288
- '- The validator creates private offline Datadog response files outside the repository and removes them ' +
289
- 'afterward.',
290
- '- The dd-trace validation path opens no listener, contacts no Datadog endpoint, requires no real Datadog ' +
291
- 'credentials, and uploads nothing.',
292
- '- Project commands are repository code and may use the network or access local resources unless the ' +
293
- 'execution environment prevents it.',
294
- '',
295
- '## Approval Command',
296
- '',
297
- `Approval details: ${inlineCode(getRepositoryRelativePath(
298
- repositoryRoot,
299
- approvalArtifacts.approvalJsonPath
300
- ))}`,
301
- '',
302
- 'Optional: independently reproduce the approval hash without running project code:',
303
- '',
304
- codeBlock(formatIndependentHashCommand(approvalArtifacts.approvalJsonPath)),
305
- '',
306
- `Expected SHA-256: ${inlineCode(approvalDigest)}`,
307
- '',
308
- ...coveredFileVerification,
309
- 'These checks confirm that the reviewed inputs have not changed since plan generation. They do not verify ' +
310
- 'where the installed `dd-trace` package came from; establish package origin separately through trusted ' +
311
- 'lockfile/integrity metadata or a verified package tarball.',
312
- '',
313
- 'Run the approved validation command:',
314
- '',
315
- codeBlock(sanitizeString(serializeApprovalCommand({
316
- argv: validatorArgv,
317
- cwd: repositoryRoot,
318
- usesShell: false,
319
- }))),
320
- '',
321
- `Working directory: ${inlineCode(repositoryRoot)}`,
322
- '',
323
- 'Approve executing the commands and temporary file operations shown in this summary?'
324
- )
325
- return lines.join('\n')
326
- }
327
-
328
- /**
329
- * Appends one runnable framework to the bounded approval summary.
330
- *
331
- * @param {string[]} lines rendered summary lines
332
- * @param {object} framework manifest framework entry
333
- * @param {string|null|undefined} requestedScenario selected scenario
334
- * @param {string} repositoryRoot repository root
335
- * @returns {void}
336
- */
337
- function appendApprovalSummaryFramework (lines, framework, requestedScenario, repositoryRoot) {
338
- const basicCommand = getBasicReportingCommand(framework)
339
- const directInitialization = getDirectInitialization(framework)
340
- const maxTestCount = framework.preflight?.maxTestCount ?? 50
341
- lines.push(`### ${plainText(formatFrameworkLabel(framework, repositoryRoot))}`, '')
342
-
343
- for (const setupCommand of framework.setup?.commands || []) {
344
- appendApprovalSummaryCommand(lines, {
345
- command: setupCommand,
346
- label: `Project setup: ${setupCommand.id || setupCommand.description || 'setup'}`,
347
- repositoryRoot,
348
- runs: '1',
349
- })
350
- }
351
- appendApprovalSummaryCommand(lines, {
352
- command: getDatadogCleanCommand(basicCommand),
353
- label: 'Test execution without Datadog',
354
- note: 'Inherited NODE_OPTIONS and DD_* variables are removed. The command must report between 1 and ' +
355
- `${maxTestCount} tests.`,
356
- repositoryRoot,
357
- runs: '1, plus 1 clean confirmation only if the Datadog run exits differently',
358
- })
359
- appendApprovalSummaryCommand(lines, {
360
- command: basicCommand,
361
- environmentOverrides: { NODE_OPTIONS: directInitialization },
362
- label: 'Test execution with Datadog',
363
- note: 'A second Datadog debug run occurs only when diagnosis needs debug output.',
364
- repositoryRoot,
365
- runs: '1, plus at most 1 Datadog debug run when needed',
366
- })
367
-
368
- const ciWiringSelected = !requestedScenario || requestedScenario === 'ci-wiring'
369
- if (ciWiringSelected && framework.ciWiringCommand) {
370
- appendApprovalSummaryCommand(lines, {
371
- command: getCiWiringCommand(framework),
372
- label: 'CI test execution',
373
- note: 'A short preload probe may run when initialization reachability needs confirmation.',
374
- repositoryRoot,
375
- runs: '1, plus at most 1 preload probe',
376
- })
377
- } else if (ciWiringSelected) {
378
- lines.push(
379
- '**CI test execution:** not run.',
380
- '',
381
- `Reason: ${plainText(
382
- framework.ciWiring?.reason || framework.ciWiring?.diagnosis || 'No replayable CI test command was selected.'
383
- )}`,
384
- ''
385
- )
386
- }
387
-
388
- const selectedGeneratedScenario = getSelectedGeneratedScenario(requestedScenario)
389
- const advancedSelected = !requestedScenario || selectedGeneratedScenario
390
- const strategy = framework.generatedTestStrategy
391
- if (advancedSelected && strategy && ['planned', 'verified'].includes(strategy.status)) {
392
- const scenarios = selectedGeneratedScenario
393
- ? (strategy.scenarios || []).filter(scenario => scenario.id === selectedGeneratedScenario)
394
- : strategy.scenarios || []
395
- for (const scenario of scenarios) {
396
- appendApprovalSummaryCommand(lines, {
397
- command: getLocalValidationCommand(framework, scenario.runCommand),
398
- environmentOverrides: { NODE_OPTIONS: directInitialization },
399
- label: GENERATED_SCENARIO_DETAILS[scenario.id]?.heading || `Advanced check: ${scenario.id}`,
400
- note: 'Runs verification, identity discovery, and feature validation; a debug run occurs only on failure.',
401
- repositoryRoot,
402
- runs: '3, or 4 when the debug run is needed',
403
- })
113
+ lines.push(`### ${plain(getFrameworkLabel(framework, root))}`, '')
114
+ if (framework.status !== 'runnable') {
115
+ lines.push(`Status: ${plain(formatStatus(framework.status))}.`)
116
+ for (const note of framework.notes || []) lines.push(`- ${plain(note)}`)
117
+ lines.push('')
118
+ continue
404
119
  }
405
120
 
406
- lines.push('**Temporary test source:**', '')
407
- for (const file of strategy.files || []) {
121
+ if (requestedScenario === 'ci-wiring') {
122
+ lines.push('Status: static CI audit only; no project test command is selected.', '')
123
+ } else {
124
+ const basic = getBasicCommand(framework)
125
+ const unavailable = getUnavailableExecutable(basic)
126
+ const status = unavailable
127
+ ? `local validation will be incomplete because ${inline(unavailable)} is unavailable`
128
+ : 'eligible for approved clean preflight; runtime prerequisites are unverified'
408
129
  lines.push(
409
- `${inlineCode(getRepositoryRelativePath(repositoryRoot, file.path))}`,
130
+ `Status: ${status}.`,
131
+ `Representative test: ${inline(relative(root, framework.validation.testFile))}`,
132
+ `Working directory: ${inline(relative(root, basic.cwd))}`,
133
+ `Timeout: ${basic.timeoutMs} ms`,
134
+ ...(Object.keys(basic.env || {}).length > 0
135
+ ? [`Runner environment: ${inline(JSON.stringify(basic.env))}`]
136
+ : []),
137
+ ...formatOmittedRunnerOptions(framework.validation.omittedRunnerOptions),
410
138
  '',
411
- codeBlock(file.contentLines.join('\n')),
139
+ 'Basic Reporting command:',
140
+ '',
141
+ codeBlock(serializeApprovalCommand(basic)),
142
+ '',
143
+ 'The command runs once without Datadog and once with validator-owned offline initialization. A debug rerun ' +
144
+ 'and one clean confirmation may run only when needed to diagnose a mismatch.',
412
145
  ''
413
146
  )
414
- }
415
- lines.push('**Files removed after validation:**', '')
416
- for (const cleanupPath of strategy.cleanupPaths || []) {
417
- lines.push(`- ${inlineCode(getRepositoryRelativePath(repositoryRoot, cleanupPath))}`)
418
- }
419
- lines.push('')
420
- } else if (advancedSelected && strategy) {
421
- lines.push(`**Advanced feature checks:** not run. ${plainText(strategy.reason || strategy.status)}`, '')
422
- }
423
- }
424
-
425
- /**
426
- * Appends one exact command and its execution-relevant context to the approval summary.
427
- *
428
- * @param {string[]} lines rendered summary lines
429
- * @param {object} input command summary
430
- * @param {object} input.command structured command
431
- * @param {Record<string, string>} [input.environmentOverrides] validator-provided readable environment
432
- * @param {string} input.label customer-facing command label
433
- * @param {string} [input.note] additional execution behavior
434
- * @param {string} input.repositoryRoot repository root
435
- * @param {string} input.runs maximum execution count
436
- * @returns {void}
437
- */
438
- function appendApprovalSummaryCommand (lines, {
439
- command,
440
- environmentOverrides = {},
441
- label,
442
- note,
443
- repositoryRoot,
444
- runs,
445
- }) {
446
- lines.push(
447
- `**${plainText(label)}**`,
448
- '',
449
- codeBlock(formatCommandForPlan(command, repositoryRoot, environmentOverrides)),
450
- '',
451
- `- Working directory: ${inlineCode(getRepositoryRelativePath(repositoryRoot, command.cwd))}`,
452
- `- Runs: ${plainText(runs)}`,
453
- `- Timeout: ${command.timeoutMs || 300_000} ms`
454
- )
455
- if (command.usesShell) lines.push(`- Shell executable: ${inlineCode(command.shell || 'platform default shell')}`)
456
- const outputPaths = getCommandOutputPaths(command)
457
- if (outputPaths.length > 0) {
458
- lines.push('- Command-created outputs removed afterward: ' + outputPaths.map(outputPath => {
459
- return inlineCode(getRepositoryRelativePath(repositoryRoot, outputPath))
460
- }).join(', '))
461
- }
462
- for (const adjustment of command.localAdjustments || []) {
463
- lines.push(`- Local adjustment: ${plainText(adjustment)}`)
464
- }
465
- if (note) lines.push(`- ${plainText(note)}`)
466
- lines.push('')
467
- }
468
-
469
- /**
470
- * Returns the durable customer-facing plan path written by --print-plan.
471
- *
472
- * @param {string} out validation output directory
473
- * @returns {string} absolute execution plan path
474
- */
475
- function getExecutionPlanPath (out) {
476
- return path.join(out, EXECUTION_PLAN_FILENAME)
477
- }
478
-
479
- /**
480
- * Returns the bounded approval summary path written by --print-plan.
481
- *
482
- * @param {string} out validation output directory
483
- * @returns {string} absolute approval summary path
484
- */
485
- function getApprovalSummaryPath (out) {
486
- return path.join(out, APPROVAL_SUMMARY_FILENAME)
487
- }
488
-
489
- /**
490
- * Refuses to render an approvable plan with a command that cannot start before setup runs.
491
- *
492
- * @param {object} manifest normalized validation manifest
493
- * @param {string|null|undefined} requestedScenario selected scenario
494
- * @returns {void}
495
- */
496
- function assertPlannedExecutablesAvailable (manifest, requestedScenario) {
497
- for (const framework of manifest.frameworks) {
498
- if (framework.status !== 'runnable') continue
499
-
500
- const plannedCommands = getPlannedCommands(framework, requestedScenario)
501
- for (const plannedCommand of plannedCommands) {
502
- const executable = getUnavailableExecutable(plannedCommand.command)
503
- if (!executable) continue
504
-
505
- throw new Error(
506
- `Cannot render an approvable plan because ${plannedCommand.label} for ${framework.id} uses ` +
507
- `executable "${executable}", which is not available from ${plannedCommand.command.cwd}. ` +
508
- 'Choose a locally available command or mark this check with its concrete setup blocker before asking ' +
509
- 'for approval.'
510
- )
511
- }
512
- for (const plannedCommand of plannedCommands) {
513
- const suitabilityError = getCommandSuitabilityError({
514
- command: plannedCommand.command,
515
- framework,
516
- label: plannedCommand.label,
517
- repositoryRoot: manifest.repository.root,
518
- })
519
- if (!suitabilityError) continue
520
- throw new Error(
521
- `Cannot render an approvable plan because ${plannedCommand.label} for ${framework.id} ${suitabilityError}`
522
- )
523
- }
524
- }
525
- }
526
-
527
- /**
528
- * Collects structured commands selected by the current plan options.
529
- *
530
- * @param {object} framework manifest framework entry
531
- * @param {string|null|undefined} requestedScenario selected scenario
532
- * @returns {{label: string, command: object}[]} planned commands
533
- */
534
- function getPlannedCommands (framework, requestedScenario) {
535
- const commands = []
536
- for (const command of framework.setup?.commands || []) {
537
- commands.push({ label: `project setup command ${command.id || command.description || ''}`.trim(), command })
538
- }
539
-
540
- commands.push({ label: 'the selected test command', command: getBasicReportingCommand(framework) })
541
-
542
- const ciWiringSelected = !requestedScenario || requestedScenario === 'ci-wiring'
543
- if (ciWiringSelected && framework.ciWiringCommand) {
544
- commands.push({ label: 'the CI test command', command: getCiWiringCommand(framework) })
545
- }
546
-
547
- const selectedGeneratedScenario = getSelectedGeneratedScenario(requestedScenario)
548
- const advancedSelected = !requestedScenario || selectedGeneratedScenario
549
- const strategy = framework.generatedTestStrategy
550
- if (advancedSelected && strategy && ['planned', 'verified'].includes(strategy.status)) {
551
- const scenarios = selectedGeneratedScenario
552
- ? (strategy.scenarios || []).filter(scenario => scenario.id === selectedGeneratedScenario)
553
- : strategy.scenarios || []
554
- for (const scenario of scenarios) {
555
- commands.push({
556
- label: `the ${scenario.id} advanced-feature command`,
557
- command: getLocalValidationCommand(framework, scenario.runCommand),
558
- })
559
- }
560
- }
561
-
562
- return commands
563
- }
564
-
565
- /**
566
- * Builds the exact validator command covered by the approval checkpoint.
567
- *
568
- * @param {object} input command options
569
- * @param {string} input.approvedPlanSha256 digest of the approved manifest and options
570
- * @param {string} input.approvalJsonPath reviewed approval JSON path
571
- * @param {string} input.repositoryRoot repository root
572
- * @returns {string[]} validator argv
573
- */
574
- function getValidatorArgv ({
575
- approvedPlanSha256,
576
- approvalJsonPath,
577
- repositoryRoot,
578
- }) {
579
- const validatorPath = getPreferredValidatorPath(repositoryRoot)
580
- return [
581
- validatorPath === VALIDATOR_PATH ? process.execPath : 'node',
582
- validatorPath,
583
- '--run-approved-plan', getRepositoryRelativePath(repositoryRoot, approvalJsonPath),
584
- '--sha256', approvedPlanSha256,
585
- ]
586
- }
587
-
588
- /**
589
- * Uses the stable package path when it resolves to this installed validator.
590
- *
591
- * @param {string} repositoryRoot repository root
592
- * @returns {string} relative package path or exact validator path
593
- */
594
- function getPreferredValidatorPath (repositoryRoot) {
595
- const directPath = path.join(repositoryRoot, 'node_modules', 'dd-trace', 'ci', 'validate-test-optimization.js')
596
- try {
597
- if (fs.realpathSync(directPath) === fs.realpathSync(VALIDATOR_PATH)) {
598
- return path.relative(repositoryRoot, directPath).split(path.sep).join('/')
599
- }
600
- } catch {}
601
- return VALIDATOR_PATH
602
- }
603
147
 
604
- /**
605
- * Returns a platform-standard command for hashing the saved approval JSON independently of the validator.
606
- *
607
- * @param {string} approvalJsonPath absolute approval JSON path
608
- * @returns {string} printable checksum command
609
- */
610
- function formatIndependentHashCommand (approvalJsonPath) {
611
- const command = process.platform === 'win32'
612
- ? { argv: ['certutil', '-hashfile', approvalJsonPath, 'SHA256'], cwd: path.dirname(approvalJsonPath) }
613
- : { argv: ['shasum', '-a', '256', approvalJsonPath], cwd: path.dirname(approvalJsonPath) }
614
- return sanitizeString(serializeApprovalCommand({ ...command, usesShell: false }))
615
- }
616
-
617
- function appendFrameworkExecutions (
618
- lines,
619
- framework,
620
- requestedScenario,
621
- repositoryRoot,
622
- out,
623
- offlineFixtureNonce
624
- ) {
625
- const basicCommand = getBasicReportingCommand(framework)
626
- const frameworkLabel = formatFrameworkLabel(framework, repositoryRoot)
627
- const directInitialization = getDirectInitialization(framework)
628
- const maxTestCount = framework.preflight?.maxTestCount ?? 50
629
- lines.push(`### ${plainText(frameworkLabel)}`, '')
630
-
631
- for (const setupCommand of framework.setup?.commands || []) {
632
- appendExecutionSection(lines, {
633
- heading: `Project Setup: ${setupCommand.id || setupCommand.description || 'Project Setup'}`,
634
- description: 'Prepares the project for the selected test command.',
635
- command: setupCommand,
636
- executions: '1',
637
- environment: 'Use the command-specific variables shown inline. No Datadog variables are added.',
638
- repositoryRoot,
639
- })
640
- }
641
- const cleanCommand = getDatadogCleanCommand(basicCommand)
642
- appendExecutionSection(lines, {
643
- heading: 'Test Execution Without Datadog',
644
- description: 'Runs the selected test command without Datadog to confirm that the tests can run normally and ' +
645
- `that it reports between 1 and ${maxTestCount} tests.`,
646
- command: cleanCommand,
647
- executions: '1, plus 1 clean confirmation only if the Datadog run exits differently',
648
- environment: `Remove inherited NODE_OPTIONS and DD_*; ${formatCommandVariableContext(cleanCommand)}`,
649
- repositoryRoot,
650
- })
651
- appendExecutionSection(lines, {
652
- heading: 'Test Execution With Datadog',
653
- description: 'Runs the same test command with Datadog initialized and checks that test data is reported.',
654
- command: basicCommand,
655
- executions: '1, plus at most 1 debug rerun when needed',
656
- environment: 'Datadog initialization is shown inline. The validator also supplies private offline response ' +
657
- 'paths and noise-suppression settings only while this check runs.',
658
- environmentOverrides: { NODE_OPTIONS: directInitialization },
659
- repositoryRoot,
660
- })
661
-
662
- const ciWiringSelected = !requestedScenario || requestedScenario === 'ci-wiring'
663
- if (ciWiringSelected && framework.ciWiringCommand) {
664
- const ciCommand = getCiWiringCommand(framework)
665
- appendExecutionSection(lines, {
666
- heading: 'CI Test Execution',
667
- description: 'Runs the test command with the environment recorded from the identified CI job and checks ' +
668
- 'whether that CI configuration initializes Datadog in the final test process.',
669
- command: ciCommand,
670
- executions: '1, plus 1 short preload probe when needed',
671
- environment: formatCiEnvironmentSummary(ciCommand),
672
- repositoryRoot,
673
- })
674
- } else if (ciWiringSelected) {
675
- lines.push(
676
- '#### CI Test Execution',
677
- '',
678
- 'Not run.',
679
- '',
680
- `- Reason: ${plainText(
681
- framework.ciWiring?.reason || framework.ciWiring?.diagnosis || 'No replayable CI test command was selected.'
682
- )}`,
683
- ''
684
- )
685
- }
686
-
687
- const strategy = framework.generatedTestStrategy
688
- const selectedGeneratedScenario = getSelectedGeneratedScenario(requestedScenario)
689
- const advancedSelected = !requestedScenario || selectedGeneratedScenario
690
- if (advancedSelected && strategy && ['planned', 'verified'].includes(strategy.status)) {
691
- const selectedScenarios = selectedGeneratedScenario
692
- ? (strategy.scenarios || []).filter(scenario => scenario.id === selectedGeneratedScenario)
693
- : strategy.scenarios || []
694
- for (const scenario of selectedScenarios) {
695
- const command = getLocalValidationCommand(framework, scenario.runCommand)
696
- const details = GENERATED_SCENARIO_DETAILS[scenario.id] || {
697
- heading: `Advanced Check: ${scenario.id}`,
698
- description: 'Runs a temporary test to verify this advanced feature.',
148
+ const selectedScenarios = getSelectedGeneratedScenarios(framework, requestedScenario)
149
+ for (const scenario of selectedScenarios) {
150
+ const command = getGeneratedCommand(framework, scenario)
151
+ const source = getGeneratedSource(framework, scenario)
152
+ lines.push(
153
+ `Advanced check ${inline(scenario.id)}:`,
154
+ '',
155
+ codeBlock(serializeApprovalCommand(command)),
156
+ '',
157
+ `Temporary file: ${inline(relative(root, scenario.testIdentities[0].file))}`,
158
+ '',
159
+ codeBlock(source),
160
+ '',
161
+ 'Execution count: one clean generated-test verification, one instrumented identity-discovery run, and ' +
162
+ 'one feature-validation run. One debug rerun may run only after a failure.',
163
+ ''
164
+ )
165
+ }
166
+ for (const file of getSelectedSupportFiles(framework, selectedScenarios)) {
167
+ lines.push(
168
+ 'Advanced check support file:',
169
+ '',
170
+ `Temporary file: ${inline(relative(root, file.path))}`,
171
+ '',
172
+ codeBlock(file.contentLines.join('\n')),
173
+ ''
174
+ )
699
175
  }
700
- appendExecutionSection(lines, {
701
- heading: details.heading,
702
- description: `${details.description} The framework runner is invoked directly so only this temporary test ` +
703
- 'runs instead of the broader project test suite.',
704
- command,
705
- executions: '3: verify the test alone, discover its identity, then validate the feature; ' +
706
- 'plus 1 debug rerun only on failure',
707
- environment: 'Datadog initialization is shown inline. The validator supplies the feature setting from a ' +
708
- 'private offline response only while this check runs.',
709
- environmentOverrides: { NODE_OPTIONS: directInitialization },
710
- repositoryRoot,
711
- })
712
176
  }
713
- }
714
-
715
- appendOfflineArtifacts(lines, {
716
- offlineFixtureNonce,
717
- framework,
718
- out,
719
- repositoryRoot,
720
- requestedScenario,
721
- })
722
177
 
723
- if (advancedSelected && strategy && ['planned', 'verified'].includes(strategy.status)) {
724
- lines.push(
725
- '#### Temporary Tests Created for Advanced Checks',
726
- '',
727
- 'The validator creates these tests temporarily and removes them after validation.',
728
- ''
729
- )
730
- for (const file of strategy.files || []) {
731
- lines.push(
732
- `##### ${inlineCode(getRepositoryRelativePath(repositoryRoot, file.path))}`,
733
- '',
734
- codeBlock(file.contentLines.join('\n')),
735
- ''
736
- )
737
- }
178
+ const ci = framework.ciWiring
738
179
  lines.push(
739
- '#### Temporary Test Cleanup',
740
- '',
741
- 'The validator removes these temporary test and state files after validation. Paths are relative to the ' +
742
- 'repository root:',
743
- ''
744
- )
745
- for (const cleanupPath of strategy.cleanupPaths || []) {
746
- lines.push(`- ${inlineCode(getRepositoryRelativePath(repositoryRoot, cleanupPath))}`)
747
- }
748
- lines.push('', 'Directories created for these files are also removed when they are empty.', '')
749
- } else if (advancedSelected && strategy) {
750
- lines.push(
751
- '#### Advanced Feature Checks',
752
- '',
753
- 'Not run.',
754
- '',
755
- `- Reason: ${plainText(strategy.reason || strategy.status)}`,
180
+ 'CI audit: static only; no CI or package command will execute.',
181
+ `CI review: ${plain(ci?.reviewComplete ? 'complete' : 'incomplete')}.`,
182
+ ...(ci?.configFile ? [`CI file: ${inline(relative(root, ci.configFile))}`] : []),
756
183
  ''
757
184
  )
758
185
  }
759
- lines.push('')
760
- }
761
186
 
762
- /**
763
- * Describes the offline Datadog responses and event outputs used by instrumented executions.
764
- *
765
- * @param {string[]} lines rendered plan lines
766
- * @param {object} input plan inputs
767
- * @param {string} input.offlineFixtureNonce random fixture-root nonce
768
- * @param {object} input.framework framework manifest entry
769
- * @param {string} input.out validation result directory
770
- * @param {string} input.repositoryRoot repository root
771
- * @param {string|null|undefined} input.requestedScenario selected scenario
772
- */
773
- function appendOfflineArtifacts (lines, {
774
- offlineFixtureNonce,
775
- framework,
776
- out,
777
- repositoryRoot,
778
- requestedScenario,
779
- }) {
780
187
  lines.push(
781
- '#### Offline Datadog Responses',
782
- '',
783
- 'During normal operation, `dd-trace` downloads Test Optimization settings and test lists from Datadog. ' +
784
- 'For this offline validation, the validator writes equivalent bounded responses to a private temporary ' +
785
- 'directory and `dd-trace` reads them from the filesystem. No Datadog backend, Agent, or network endpoint ' +
786
- 'is used.',
787
- ''
788
- )
789
-
790
- const scenarioNames = getOfflineScenarioNames(requestedScenario)
791
- const firstFixture = getOfflineFixturePaths({
792
- offlineFixtureNonce,
793
- framework,
794
- scenarioName: scenarioNames[0],
795
- })
796
- const frameworkFixtureRoot = path.dirname(firstFixture.root)
797
- lines.push(
798
- `Private response directory: ${inlineCode(frameworkFixtureRoot)}`,
799
- '',
800
- 'Each check gets an isolated subdirectory containing bounded Test Optimization settings and test lists that ' +
801
- '`dd-trace` normally receives from Datadog. Isolation prevents a baseline, feature check, or conditional ' +
802
- 'debug rerun from overwriting another check. A debug rerun uses the same response data and adds ' +
803
- '`DD_TRACE_DEBUG=1`.',
188
+ '## Writes and Cleanup',
189
+ '',
190
+ `- Validation artifacts: ${inline(relative(root, out))}`,
191
+ ...(requestedScenario === 'ci-wiring'
192
+ ? []
193
+ : [
194
+ '- Temporary generated tests shown above are created only for advanced checks and removed afterward.',
195
+ '- Declared framework output is refused if it already exists and removed after each command.',
196
+ '- Private offline fixtures are created outside the repository and removed after each check.',
197
+ ]),
198
+ '',
199
+ '## Security Boundary',
200
+ '',
201
+ '- The validation transport opens no listener, contacts no Datadog endpoint, and uses no real Datadog ' +
202
+ 'credentials.',
203
+ ...(requestedScenario === 'ci-wiring'
204
+ ? ['- This CI-only plan executes no project test command.']
205
+ : [
206
+ '- Project runners and tests are arbitrary repository code. Approval means trusting the listed files, ' +
207
+ 'their imported code, and any subprocesses they start.',
208
+ ]),
209
+ '- The approval digest binds the manifest, validator package, Node.js binary, runner, selected test, config ' +
210
+ 'files, CI evidence file, generated source, options, and output directory.',
211
+ '- Missing dependencies, unsupported launchers, arbitrary wrapper chains, dynamic CI, and ambiguous evidence ' +
212
+ 'produce an incomplete result.',
213
+ '',
214
+ 'Approval details:',
215
+ '',
216
+ `- JSON: ${inline(relative(root, approvalArtifacts.approvalJsonPath))}`,
217
+ `- Checksums: ${inline(relative(root, approvalArtifacts.coveredFilesPath))}`,
218
+ `- SHA-256: ${inline(approvalArtifacts.digest)}`,
219
+ '',
220
+ 'Run exactly this command after approval:',
221
+ '',
222
+ codeBlock(serializeApprovalCommand({
223
+ argv: validatorArgv,
224
+ cwd: root,
225
+ usesShell: false,
226
+ })),
804
227
  '',
805
- `Captured event artifacts: ${inlineCode(getRepositoryRelativePath(
806
- repositoryRoot,
807
- path.join(out, 'runs', getArtifactId(framework.id))
808
- ))}`,
228
+ `Working directory: ${inline(root)}`,
809
229
  '',
810
- 'Each execution writes bounded temporary JSON payload files under `.offline-payloads/payloads/tests/`, using ' +
811
- 'the Test Optimization payload-file layout. The temporary payload directory is removed after parsing, and a ' +
812
- 'sanitized `events.ndjson` file remains for diagnosis. Exact fixture recipes and paths are included in the ' +
813
- 'approval digest even though this plan summarizes their shared layout.',
814
- ''
230
+ 'If an agent platform hard-denies the exact command, do not alter it or broaden permissions. Run this same ' +
231
+ 'command in a normal project terminal and ask the agent to interpret the generated report.'
815
232
  )
233
+ return lines.join('\n')
816
234
  }
817
235
 
818
- /**
819
- * Renders one customer-facing validation step with its exact command and execution context.
820
- *
821
- * @param {string[]} lines rendered plan lines
822
- * @param {object} input execution details
823
- * @param {string} input.heading customer-facing check name
824
- * @param {string} input.description reason for the execution
825
- * @param {object} input.command manifest command
826
- * @param {string} input.executions maximum execution count
827
- * @param {string} input.environment environment changes made for this check
828
- * @param {Record<string, string>} [input.environmentOverrides] readable validator-provided environment values
829
- * @param {string} input.repositoryRoot repository root
830
- * @returns {void}
831
- */
832
- function appendExecutionSection (lines, {
833
- heading,
834
- description,
835
- command,
836
- executions,
837
- environment,
838
- environmentOverrides = {},
839
- repositoryRoot,
840
- }) {
841
- lines.push(
842
- `#### ${plainText(heading)}`,
843
- '',
844
- plainText(description),
845
- '',
846
- 'Command:',
847
- '',
848
- codeBlock(formatCommandForPlan(command, repositoryRoot, environmentOverrides)),
849
- '',
850
- `- Working directory: ${inlineCode(getRepositoryRelativePath(repositoryRoot, command.cwd))}`,
851
- `- Runs: ${plainText(executions)}`,
852
- `- Environment changes: ${plainText(environment)}`,
853
- `- Timeout: ${command.timeoutMs || 300_000} ms`
854
- )
855
- if (command.usesShell) lines.push(`- Shell executable: ${inlineCode(command.shell || 'platform default shell')}`)
856
- const outputPaths = getCommandOutputPaths(command)
857
- if (outputPaths.length > 0) {
858
- lines.push('- Command-created outputs: ' + outputPaths.map(outputPath => {
859
- return inlineCode(getRepositoryRelativePath(repositoryRoot, outputPath))
860
- }).join(', ') + ' (must not exist before validation; newly created paths are removed)')
861
- }
862
- for (const adjustment of command.localAdjustments || []) {
863
- lines.push(`- Local adjustment: ${plainText(adjustment)}`)
864
- }
865
- lines.push('')
236
+ function formatOmittedRunnerOptions (options = []) {
237
+ return options.map(option => option === '--run'
238
+ ? 'Normalized runner option: `--run` is omitted because the validator supplies Vitest `run` itself.'
239
+ : 'Omitted runner option: `--typecheck` is excluded because validation executes runtime tests only.')
866
240
  }
867
241
 
868
242
  /**
869
- * Shortens a validated repository path for customer-facing plans.
243
+ * Selects generated scenarios covered by a plan.
870
244
  *
871
- * @param {string} repositoryRoot repository root shown at the start of the plan
872
- * @param {string} filename absolute validated path
873
- * @returns {string} repository-relative path when possible
245
+ * @param {object} framework framework entry
246
+ * @param {string|null} requestedScenario selected scenario
247
+ * @returns {object[]} selected generated scenarios
874
248
  */
875
- function getRepositoryRelativePath (repositoryRoot, filename) {
876
- const relative = path.relative(repositoryRoot, filename)
877
- if (!relative) return '.'
878
- if (relative.startsWith('..') || path.isAbsolute(relative)) return filename
879
- return relative.split(path.sep).join('/')
880
- }
881
-
882
- function getSelectedGeneratedScenario (requestedScenario) {
883
- return {
884
- efd: 'basic-pass',
885
- atr: 'atr-fail-once',
886
- 'test-management': 'test-management-target',
887
- }[requestedScenario]
249
+ function getSelectedGeneratedScenarios (framework, requestedScenario) {
250
+ if (requestedScenario === 'basic-reporting' || requestedScenario === 'ci-wiring') return []
251
+ const scenarios = framework.generatedTestStrategy?.scenarios || []
252
+ if (!requestedScenario) return scenarios
253
+ const id = SCENARIO_TO_GENERATED_ID[requestedScenario]
254
+ return id ? scenarios.filter(scenario => scenario.id === id) : []
888
255
  }
889
256
 
890
257
  /**
891
- * Describes command-specific variables without displaying their values.
258
+ * Returns adapter support files required by selected generated scenarios.
892
259
  *
893
- * @param {object} command manifest command
894
- * @returns {string} variable context
260
+ * @param {object} framework framework entry
261
+ * @param {object[]} selectedScenarios selected generated scenarios
262
+ * @returns {object[]} support files
895
263
  */
896
- function formatCommandVariableContext (command) {
897
- const names = Object.keys(command.env || {})
898
- return names.length > 0
899
- ? `keep command variables: ${names.join(', ')}`
900
- : 'the command sets no other environment variables'
264
+ function getSelectedSupportFiles (framework, selectedScenarios) {
265
+ if (selectedScenarios.length === 0) return []
266
+ const scenarioPaths = new Set((framework.generatedTestStrategy?.scenarios || []).map(scenario => {
267
+ return path.resolve(scenario.testIdentities[0].file)
268
+ }))
269
+ return (framework.generatedTestStrategy?.files || []).filter(file => {
270
+ return !scenarioPaths.has(path.resolve(file.path))
271
+ })
901
272
  }
902
273
 
903
274
  /**
904
- * Renders the command and its command-specific environment in one readable block.
275
+ * Returns canonical generated source for a scenario.
905
276
  *
906
- * @param {object} command manifest command
907
- * @param {string} repositoryRoot absolute repository root
908
- * @param {Record<string, string>} environmentOverrides readable validator-provided values
909
- * @returns {string} readable command
277
+ * @param {object} framework framework entry
278
+ * @param {object} scenario generated scenario
279
+ * @returns {string} source
910
280
  */
911
- function formatCommandForPlan (command, repositoryRoot, environmentOverrides) {
912
- const displayCommand = command.usesShell
913
- ? command
914
- : {
915
- ...command,
916
- argv: command.argv.map((argument, index) => {
917
- return formatCommandArgument(argument, index, command, repositoryRoot)
918
- }),
919
- }
920
- const environment = { ...command.env, ...environmentOverrides }
921
- if (command.env?.NODE_OPTIONS && environmentOverrides.NODE_OPTIONS) {
922
- environment.NODE_OPTIONS = `${environmentOverrides.NODE_OPTIONS} ${command.env.NODE_OPTIONS}`
923
- }
924
- const sanitizedEnvironment = sanitizeEnv(environment) || {}
925
- const prefix = Object.entries(sanitizedEnvironment).map(([name, value]) => {
926
- return `${name}=${formatEnvironmentValue(value)}`
927
- }).join(' ')
928
- const serialized = sanitizeString(serializeApprovalCommand(displayCommand))
929
- return prefix ? `${prefix} ${serialized}` : serialized
281
+ function getGeneratedSource (framework, scenario) {
282
+ const filename = path.resolve(scenario.testIdentities[0].file)
283
+ const file = framework.generatedTestStrategy.files.find(candidate => path.resolve(candidate.path) === filename)
284
+ return file?.contentLines?.join('\n') || '<generated source unavailable>'
930
285
  }
931
286
 
932
287
  /**
933
- * Shortens repository-contained command paths without changing their meaning from the stated working directory.
288
+ * Returns the execution plan path.
934
289
  *
935
- * @param {string} argument command argument
936
- * @param {number} index argument index
937
- * @param {object} command manifest command
938
- * @param {string} repositoryRoot absolute repository root
939
- * @returns {string} readable argument
290
+ * @param {string} out output directory
291
+ * @returns {string} plan path
940
292
  */
941
- function formatCommandArgument (argument, index, command, repositoryRoot) {
942
- if (index === 0 && path.resolve(argument) === path.resolve(process.execPath)) return 'node'
943
- if (!path.isAbsolute(argument) || !isPathInside(repositoryRoot, argument)) return argument
944
- return path.relative(command.cwd, argument) || '.'
293
+ function getExecutionPlanPath (out) {
294
+ return path.join(out, EXECUTION_PLAN_FILENAME)
945
295
  }
946
296
 
947
297
  /**
948
- * Quotes one environment value only when its characters require it.
298
+ * Formats a framework label.
949
299
  *
950
- * @param {string} value environment value
951
- * @returns {string} readable assignment value
300
+ * @param {object} framework framework entry
301
+ * @param {string} root repository root
302
+ * @returns {string} label
952
303
  */
953
- function formatEnvironmentValue (value) {
954
- return serializeApprovalCommand({ argv: [String(value)], usesShell: false })
304
+ function getFrameworkLabel (framework, root) {
305
+ const project = framework.project?.name || relative(root, framework.project?.root || root)
306
+ return `${framework.framework} in ${project || 'root project'}`
955
307
  }
956
308
 
957
309
  /**
958
- * Checks whether a path remains within a parent directory.
310
+ * Formats a framework status.
959
311
  *
960
- * @param {string} parent parent directory
961
- * @param {string} child candidate child path
962
- * @returns {boolean} whether child is within parent
312
+ * @param {string} status status id
313
+ * @returns {string} status text
963
314
  */
964
- function isPathInside (parent, child) {
965
- const relative = path.relative(parent, child)
966
- return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative))
315
+ function formatStatus (status) {
316
+ return {
317
+ detected_not_runnable: 'detected but no direct target was selected',
318
+ requires_manual_setup: 'requires normal project setup',
319
+ unsupported_by_validator: 'unsupported by this validator',
320
+ }[status] || status
967
321
  }
968
322
 
969
323
  /**
970
- * Returns the customer-facing Datadog preload required by one framework.
324
+ * Formats a repository-relative path.
971
325
  *
972
- * @param {object} framework manifest framework entry
973
- * @returns {string} NODE_OPTIONS value
326
+ * @param {string} root repository root
327
+ * @param {string} filename absolute path
328
+ * @returns {string} relative display path
974
329
  */
975
- function getDirectInitialization (framework) {
976
- return framework.framework === 'vitest'
977
- ? '--import dd-trace/register.js -r dd-trace/ci/init'
978
- : '-r dd-trace/ci/init'
330
+ function relative (root, filename) {
331
+ const value = path.relative(root, filename)
332
+ return value || '.'
979
333
  }
980
334
 
981
335
  /**
982
- * Describes the variables captured from the selected CI job.
336
+ * Formats a safe Markdown code block.
983
337
  *
984
- * @param {object} command CI test command
985
- * @returns {string} customer-facing environment summary
338
+ * @param {string} value text
339
+ * @returns {string} code block
986
340
  */
987
- function formatCiEnvironmentSummary (command) {
988
- const names = Object.keys(command.env || {})
989
- const datadogNames = names.filter(name => name.startsWith('DD_') || name === 'NODE_OPTIONS')
990
- if (datadogNames.length === 0) {
991
- return 'The selected CI job supplies no Datadog variables. Other recorded CI variables, if any, are shown ' +
992
- 'inline.'
993
- }
994
- return 'Variables recorded from the selected CI job are shown inline. Secret-like values are redacted.'
341
+ function codeBlock (value) {
342
+ return `\`\`\`text\n${plainMultiline(value).replaceAll('```', String.raw`\u0060\u0060\u0060`)}\n\`\`\``
995
343
  }
996
344
 
997
345
  /**
998
- * Names a framework using the package or project that contributors recognize.
346
+ * Formats safe inline code.
999
347
  *
1000
- * @param {object} framework manifest framework entry
1001
- * @param {string} repositoryRoot absolute repository root
1002
- * @returns {string} customer-facing framework label
348
+ * @param {string} value text
349
+ * @returns {string} inline code
1003
350
  */
1004
- function formatFrameworkLabel (framework, repositoryRoot) {
1005
- const frameworkName = FRAMEWORK_NAMES[framework.framework] || framework.framework || 'Test'
1006
- const projectName = framework.project?.name
1007
- if (projectName && projectName !== 'root') return `${frameworkName} tests for ${projectName}`
1008
- const projectRoot = framework.project?.root
1009
- const relativeRoot = projectRoot && getRepositoryRelativePath(repositoryRoot, projectRoot)
1010
- return relativeRoot && relativeRoot !== '.'
1011
- ? `${frameworkName} tests in ${relativeRoot}`
1012
- : `${frameworkName} tests`
351
+ function inline (value) {
352
+ return `\`${plain(value).replaceAll('`', String.raw`\u0060`)}\``
1013
353
  }
1014
354
 
1015
355
  /**
1016
- * Lists each executable identity once instead of repeating it under every command.
356
+ * Sanitizes one line of plan text.
1017
357
  *
1018
- * @param {string[]} lines rendered plan lines
1019
- * @param {object} manifest normalized validation manifest
1020
- * @param {string|null|undefined} requestedScenario selected scenario
1021
- * @returns {void}
358
+ * @param {unknown} value text
359
+ * @returns {string} safe text
1022
360
  */
1023
- function appendCommandIntegrity (lines, manifest, requestedScenario) {
1024
- const executables = new Map()
1025
- for (const framework of manifest.frameworks) {
1026
- if (framework.status !== 'runnable') continue
1027
-
1028
- for (const { command } of getPlannedCommands(framework, requestedScenario)) {
1029
- const executable = getApprovedExecutable(command)
1030
- if (executable) {
1031
- for (const identity of [executable, ...(executable.delegated || [])]) {
1032
- const key = `${identity.invocationPath}:${identity.path}:${identity.sha256}`
1033
- const entry = executables.get(key) || { executable: identity, labels: new Set() }
1034
- entry.labels.add(getExecutableLabel(command, identity.invocationPath))
1035
- executables.set(key, entry)
1036
- }
1037
- }
1038
- }
1039
- }
1040
- if (executables.size === 0) return
1041
-
1042
- lines.push(
1043
- '## Executables Used',
1044
- '',
1045
- 'These programs start the commands shown above. The validator records their fingerprints internally and ' +
1046
- 'stops if an executable or PATH selection changes after approval. This confirms that the approved programs ' +
1047
- 'did not change; it does not establish that project scripts, packages, or subprocesses are safe.',
1048
- ''
1049
- )
1050
- for (const { executable, labels } of executables.values()) {
1051
- const canonicalTarget = executable.invocationPath === executable.path
1052
- ? ''
1053
- : ` (verified target: ${inlineCode(executable.path)})`
1054
- lines.push(`- ${[...labels].sort().join(', ')}: ${inlineCode(executable.invocationPath)}${canonicalTarget}`)
1055
- }
1056
- lines.push('')
1057
- }
1058
-
1059
- function getExecutableLabel (command, invocationPath) {
1060
- const name = path.basename(invocationPath).replace(/\.(?:bat|cmd|exe)$/i, '').toLowerCase()
1061
- if (command.usesShell) {
1062
- return {
1063
- bash: 'Bash shell',
1064
- sh: 'POSIX shell',
1065
- zsh: 'Zsh shell',
1066
- }[name] || `${name} shell`
1067
- }
1068
- return {
1069
- bash: 'Bash shell',
1070
- node: 'Node.js',
1071
- npm: 'npm',
1072
- npx: 'npx',
1073
- pnpm: 'pnpm',
1074
- sh: 'POSIX shell',
1075
- yarn: 'Yarn',
1076
- zsh: 'Zsh shell',
1077
- }[name] || name
361
+ function plain (value) {
362
+ return sanitizeString(String(value ?? '')).replaceAll(/\p{Cc}+/gu, ' ').trim()
1078
363
  }
1079
364
 
1080
365
  /**
1081
- * Converts manifest framework statuses into customer-facing plan text.
366
+ * Sanitizes multiline plan text.
1082
367
  *
1083
- * @param {string} status manifest framework status
1084
- * @returns {string} customer-facing status
368
+ * @param {unknown} value text
369
+ * @returns {string} safe text
1085
370
  */
1086
- function formatFrameworkStatus (status) {
1087
- return {
1088
- runnable: 'will be validated',
1089
- detected_not_runnable: 'detected, but no runnable command was found',
1090
- requires_external_service: 'requires an external service',
1091
- requires_manual_setup: 'requires additional setup',
1092
- unsupported_by_validator: 'not supported by this validator',
1093
- unknown: 'could not be determined',
1094
- }[status] || plainText(status)
371
+ function plainMultiline (value) {
372
+ return sanitizeString(String(value ?? '')).replaceAll('\r\n', '\n').replaceAll('\r', '\n').trim()
1095
373
  }
1096
374
 
1097
- function codeBlock (value) {
1098
- return `\`\`\`text\n${visibleMultilineText(value).replaceAll('```', String.raw`\u0060\u0060\u0060`)}\n\`\`\``
1099
- }
1100
-
1101
- function inlineCode (value) {
1102
- return `\`${plainText(value).replaceAll('`', String.raw`\u0060`)}\``
1103
- }
1104
-
1105
- function plainText (value) {
1106
- return sanitizeString(String(value ?? '')).replaceAll(CONTROL_CHARACTERS_PATTERN, ' ').trim()
1107
- }
1108
-
1109
- function visibleMultilineText (value) {
1110
- return String(value ?? '')
1111
- .replaceAll('\r\n', '\n')
1112
- .replaceAll('\r', String.raw`\r`)
1113
- // eslint-disable-next-line prefer-regex-literals
1114
- .replaceAll(new RegExp(String.raw`[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]`, 'g'), character => {
1115
- return String.raw`\u${character.charCodeAt(0).toString(16).padStart(4, '0')}`
1116
- })
1117
- .trim()
375
+ module.exports = {
376
+ formatExecutionPlan,
377
+ formatExecutionPlanArtifacts,
378
+ getExecutionPlanPath,
1118
379
  }
1119
-
1120
- module.exports = { formatExecutionPlan, getApprovalSummaryPath, getExecutionPlanPath }