dd-trace 5.129.0 → 5.131.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/LICENSE-3rdparty.csv +1 -0
  2. package/ci/vitest-no-worker-init-setup.mjs +22 -12
  3. package/index.d.ts +166 -24
  4. package/package.json +7 -7
  5. package/packages/datadog-esbuild/index.js +9 -6
  6. package/packages/datadog-instrumentations/src/ai.js +26 -0
  7. package/packages/datadog-instrumentations/src/anthropic.js +111 -9
  8. package/packages/datadog-instrumentations/src/aws-sdk.js +2 -2
  9. package/packages/datadog-instrumentations/src/claude-agent-sdk.js +5 -1
  10. package/packages/datadog-instrumentations/src/cucumber.js +40 -4
  11. package/packages/datadog-instrumentations/src/helpers/hooks.js +0 -7
  12. package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentation-registry.js +2 -2
  13. package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/ai.js +25 -0
  14. package/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/playwright.js +8 -0
  15. package/packages/datadog-instrumentations/src/helpers/rewriter/targets.json +2 -0
  16. package/packages/datadog-instrumentations/src/jest/session-error.js +102 -0
  17. package/packages/datadog-instrumentations/src/jest.js +3 -10
  18. package/packages/datadog-instrumentations/src/playwright.js +26 -0
  19. package/packages/datadog-instrumentations/src/vitest-main.js +4 -1
  20. package/packages/datadog-instrumentations/src/vitest-worker.js +34 -3
  21. package/packages/datadog-plugin-aws-sdk/src/services/bedrockruntime/utils.js +83 -29
  22. package/packages/datadog-plugin-cucumber/src/index.js +19 -4
  23. package/packages/datadog-plugin-cypress/src/cypress-plugin.js +7 -0
  24. package/packages/datadog-plugin-fetch/src/index.js +3 -0
  25. package/packages/datadog-plugin-openai/src/stream-helpers.js +50 -1
  26. package/packages/datadog-plugin-oracledb/src/connection-parser.js +3 -1
  27. package/packages/datadog-plugin-undici/src/index.js +5 -0
  28. package/packages/dd-trace/src/aiguard/client.js +38 -19
  29. package/packages/dd-trace/src/aiguard/integrations/anthropic.js +29 -4
  30. package/packages/dd-trace/src/aiguard/integrations/index.js +1 -1
  31. package/packages/dd-trace/src/aiguard/integrations/openai.js +20 -56
  32. package/packages/dd-trace/src/aiguard/integrations/stream.js +60 -0
  33. package/packages/dd-trace/src/aiguard/messages/anthropic.js +64 -0
  34. package/packages/dd-trace/src/config/generated-config-types.d.ts +8 -4
  35. package/packages/dd-trace/src/config/remote_config.js +12 -0
  36. package/packages/dd-trace/src/config/supported-configurations.json +26 -5
  37. package/packages/dd-trace/src/constants.js +2 -0
  38. package/packages/dd-trace/src/debugger/constants.js +13 -4
  39. package/packages/dd-trace/src/debugger/devtools_client/breakpoints.js +14 -1
  40. package/packages/dd-trace/src/debugger/devtools_client/condition.js +109 -6
  41. package/packages/dd-trace/src/debugger/devtools_client/config.js +2 -0
  42. package/packages/dd-trace/src/debugger/devtools_client/index.js +65 -6
  43. package/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js +22 -10
  44. package/packages/dd-trace/src/debugger/devtools_client/remote_config.js +12 -8
  45. package/packages/dd-trace/src/debugger/devtools_client/send.js +14 -0
  46. package/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js +43 -5
  47. package/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js +3 -8
  48. package/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js +2 -122
  49. package/packages/dd-trace/src/debugger/guardrail-metrics.js +0 -2
  50. package/packages/dd-trace/src/debugger/index.js +60 -28
  51. package/packages/dd-trace/src/debugger/inspect-segment.js +92 -23
  52. package/packages/dd-trace/src/debugger/pause-duration-histogram.js +74 -0
  53. package/packages/dd-trace/src/debugger/probe_sampler.js +192 -52
  54. package/packages/dd-trace/src/debugger/redaction.js +139 -0
  55. package/packages/dd-trace/src/knuth-hash.js +17 -0
  56. package/packages/dd-trace/src/llmobs/constants/tags.js +33 -8
  57. package/packages/dd-trace/src/llmobs/experiments/client.js +66 -2
  58. package/packages/dd-trace/src/llmobs/experiments/evaluator.js +138 -0
  59. package/packages/dd-trace/src/llmobs/experiments/experiment.js +213 -90
  60. package/packages/dd-trace/src/llmobs/experiments/index.js +40 -1
  61. package/packages/dd-trace/src/llmobs/experiments/noop.js +53 -2
  62. package/packages/dd-trace/src/llmobs/experiments/remote-evaluator.js +110 -0
  63. package/packages/dd-trace/src/llmobs/experiments/util.js +34 -6
  64. package/packages/dd-trace/src/llmobs/gen-ai-tags.js +127 -0
  65. package/packages/dd-trace/src/llmobs/plugins/ai/ddTelemetry.js +18 -0
  66. package/packages/dd-trace/src/llmobs/plugins/ai/vercelTelemetry.js +33 -8
  67. package/packages/dd-trace/src/llmobs/plugins/anthropic/index.js +77 -41
  68. package/packages/dd-trace/src/llmobs/plugins/base.js +154 -15
  69. package/packages/dd-trace/src/llmobs/plugins/bedrockruntime.js +162 -10
  70. package/packages/dd-trace/src/llmobs/plugins/claude-agent-sdk/index.js +65 -16
  71. package/packages/dd-trace/src/llmobs/plugins/genai/index.js +14 -0
  72. package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/chat_model.js +34 -35
  73. package/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js +10 -0
  74. package/packages/dd-trace/src/llmobs/plugins/langchain/index.js +25 -2
  75. package/packages/dd-trace/src/llmobs/plugins/langgraph/index.js +8 -7
  76. package/packages/dd-trace/src/llmobs/plugins/openai/index.js +15 -2
  77. package/packages/dd-trace/src/llmobs/plugins/openai/realtime.js +5 -0
  78. package/packages/dd-trace/src/llmobs/plugins/vertexai.js +7 -0
  79. package/packages/dd-trace/src/llmobs/prompts/prompt.js +2 -1
  80. package/packages/dd-trace/src/llmobs/sdk.js +19 -3
  81. package/packages/dd-trace/src/llmobs/span_processor.js +52 -91
  82. package/packages/dd-trace/src/llmobs/tagger.js +37 -36
  83. package/packages/dd-trace/src/llmobs/writers/base.js +1 -6
  84. package/packages/dd-trace/src/openfeature/constants/constants.js +10 -0
  85. package/packages/dd-trace/src/openfeature/flagging_provider.js +14 -1
  86. package/packages/dd-trace/src/openfeature/writers/base.js +39 -18
  87. package/packages/dd-trace/src/openfeature/writers/flag-eval-evp-hook.js +105 -0
  88. package/packages/dd-trace/src/openfeature/writers/flag-evaluation-aggregation.js +190 -0
  89. package/packages/dd-trace/src/openfeature/writers/flag-evaluation-consumer.js +233 -0
  90. package/packages/dd-trace/src/openfeature/writers/flag-evaluation-context.js +262 -0
  91. package/packages/dd-trace/src/openfeature/writers/flag-evaluation-payload.js +181 -0
  92. package/packages/dd-trace/src/openfeature/writers/flag-evaluation-pii.js +108 -0
  93. package/packages/dd-trace/src/openfeature/writers/flag-evaluation-telemetry.js +154 -0
  94. package/packages/dd-trace/src/openfeature/writers/flag-evaluation-worker.js +77 -0
  95. package/packages/dd-trace/src/openfeature/writers/flag-evaluations.js +398 -0
  96. package/packages/dd-trace/src/openfeature/writers/util.js +5 -4
  97. package/packages/dd-trace/src/opentelemetry/context_manager.js +3 -0
  98. package/packages/dd-trace/src/opentelemetry/span_context.js +6 -2
  99. package/packages/dd-trace/src/opentelemetry/trace/index.js +5 -0
  100. package/packages/dd-trace/src/opentelemetry/trace/otlp_transformer.js +6 -0
  101. package/packages/dd-trace/src/opentelemetry/tracer.js +16 -18
  102. package/packages/dd-trace/src/opentracing/propagation/text_map.js +116 -25
  103. package/packages/dd-trace/src/opentracing/propagation/tracestate.js +80 -16
  104. package/packages/dd-trace/src/opentracing/span.js +14 -9
  105. package/packages/dd-trace/src/opentracing/tracer.js +10 -3
  106. package/packages/dd-trace/src/otel-sampling.js +171 -0
  107. package/packages/dd-trace/src/plugins/index.js +0 -1
  108. package/packages/dd-trace/src/plugins/util/test.js +4 -0
  109. package/packages/dd-trace/src/priority_sampler.js +155 -33
  110. package/packages/dd-trace/src/profiler.js +51 -5
  111. package/packages/dd-trace/src/profiling/profiler.js +24 -20
  112. package/packages/dd-trace/src/remote_config/capabilities.js +9 -0
  113. package/packages/dd-trace/src/ritm.js +12 -9
  114. package/packages/dd-trace/src/sampler.js +2 -5
  115. package/packages/dd-trace/src/sampling_rule.js +4 -8
  116. package/packages/dd-trace/src/span_format.js +6 -0
  117. package/packages/dd-trace/src/span_processor.js +32 -3
  118. package/packages/dd-trace/src/standalone/index.js +5 -4
  119. package/packages/dd-trace/src/standalone/tracesource_priority_sampler.js +25 -5
  120. package/packages/dd-trace/src/telemetry/metrics.js +4 -3
  121. package/vendor/dist/@datadog/openfeature-node-server/index.js +1 -1
  122. package/packages/datadog-instrumentations/src/postgres.js +0 -7
  123. package/packages/datadog-instrumentations/src/supabase.js +0 -15
@@ -1,5 +1,6 @@
1
1
  "component","origin","license","copyright"
2
2
  "@apm-js-collab/code-transformer","https://github.com/nodejs/orchestrion-js","['Apache-2.0']","['nodejs']"
3
+ "@bufbuild/protobuf","https://github.com/bufbuild/protobuf-es","['(Apache-2.0 AND BSD-3-Clause)']","['bufbuild']"
3
4
  "@datadog/flagging-core","https://github.com/DataDog/openfeature-js-client","['Apache-2.0']","['DataDog']"
4
5
  "@datadog/libdatadog","https://github.com/DataDog/libdatadog-nodejs","['Apache-2.0']","['DataDog']"
5
6
  "@datadog/libdatadog-extras","https://github.com/DataDog/libdatadog-nodejs","['Apache-2.0']","['Datadog Inc.']"
@@ -114,14 +114,12 @@ if (isNoWorkerInitActive) {
114
114
  onTestFinished(() => {
115
115
  recordTestAttemptTiming(task, attemptIndex, attemptStart)
116
116
  recordRetryErrorCount(task)
117
- if (
118
- (isAttemptToFixTest || isEarlyFlakeDetectionTestAttempt || isQuarantinedTest) &&
119
- attemptIndex === getFinalAttemptIndex(task)
120
- ) {
117
+ if (isEarlyFlakeDetectionTestAttempt || isQuarantinedTest ||
118
+ (isAttemptToFixTest && attemptIndex === getFinalAttemptIndex(task))) {
121
119
  if (isAttemptToFixTest || isEarlyFlakeDetectionTestAttempt) {
122
- recordTestOptimizationStatus(task, attemptIndex, true)
120
+ recordTestOptimizationStatus(task, attemptIndex, attemptIndex === getFinalAttemptIndex(task))
123
121
  }
124
- switchQuarantinedFinalFailure(task, attemptIndex)
122
+ switchQuarantinedFinalFailure(task)
125
123
  }
126
124
  finishRumCorrelation(task, attemptIndex)
127
125
  })
@@ -134,7 +132,7 @@ if (isNoWorkerInitActive) {
134
132
  recordTestOptimizationStatus(task, attemptIndex)
135
133
  }
136
134
  if (!restoredEarlyFlakeDetectionResult) {
137
- switchQuarantinedFinalFailure(task, attemptIndex)
135
+ switchQuarantinedFinalFailure(task)
138
136
  }
139
137
  })
140
138
  }
@@ -433,7 +431,10 @@ function recordEarlyFlakeDetectionStatus (task, attemptIndex, onlyIfNewErrors) {
433
431
  return
434
432
  }
435
433
 
436
- if (!earlyFlakeDetectionRetriesByTask.has(task)) {
434
+ // onTestFinished runs before aroundEach fixture teardown. Select the budget
435
+ // when the next attempt records the completed attempt instead.
436
+ const isCompletedAttempt = attemptIndex < task.meta.__ddTestOptCurrentAttemptIndex
437
+ if (isCompletedAttempt && !earlyFlakeDetectionRetriesByTask.has(task)) {
437
438
  const retryCount = getEarlyFlakeDetectionRetryCount(task)
438
439
  earlyFlakeDetectionRetriesByTask.set(task, retryCount)
439
440
  task.repeats = retryCount
@@ -452,9 +453,15 @@ function recordEarlyFlakeDetectionStatus (task, attemptIndex, onlyIfNewErrors) {
452
453
  )
453
454
  task.meta.__ddTestOptEfdErrorCounts[attemptIndex] = task.result?.errors?.length || 0
454
455
 
455
- if (attemptIndex === getEarlyFlakeDetectionRetryCountForTask(task) &&
456
+ // Vitest 5.0.3 retains failures across repeats. Defer EFD's overall failure until
457
+ // the last attempt, while keeping each attempt's actual outcome in the metadata.
458
+ if (attemptIndex < getEarlyFlakeDetectionRetryCountForTask(task) ||
456
459
  task.meta.__ddTestOptEfdStatuses.includes('pass')) {
457
460
  task.result.state = 'pass'
461
+ } else if (isCompletedAttempt) {
462
+ // A zero retry budget cancels repeats after we suppressed the initial failure.
463
+ task.result.state = 'fail'
464
+ markQuarantinedFailure(task)
458
465
  }
459
466
  }
460
467
 
@@ -525,7 +532,7 @@ function wrapRetryCondition (task) {
525
532
  shouldRetry = condition(error)
526
533
  }
527
534
 
528
- if (!shouldRetry && (task.result?.repeatCount || 0) >= (task.repeats || 0)) {
535
+ if (!shouldRetry) {
529
536
  const attemptIndex = task.meta.__ddTestOptCurrentAttemptIndex
530
537
  recordTestOptimizationStatus(task, attemptIndex)
531
538
  markQuarantinedFailure(task)
@@ -579,7 +586,7 @@ function getFinalAttemptIndex (task) {
579
586
  return attemptIndex + retriesRemaining + (repeatsRemaining * (retryLimit + 1))
580
587
  }
581
588
 
582
- function switchQuarantinedFinalFailure (task, attemptIndex) {
589
+ function switchQuarantinedFinalFailure (task) {
583
590
  const testSuite = getTestSuite(task)
584
591
  const testName = getTestName(task)
585
592
  if (
@@ -590,7 +597,10 @@ function switchQuarantinedFinalFailure (task, attemptIndex) {
590
597
  return
591
598
  }
592
599
 
593
- if (attemptIndex < getFinalAttemptIndex(task)) {
600
+ // Finish the retries in this repetition before suppressing its failure. A later
601
+ // repetition cannot undo a failure that Vitest has already aggregated.
602
+ const retryAttemptIndex = retryAttemptIndexByTask.get(task)?.index || 0
603
+ if (retryAttemptIndex < getRetryLimit(task)) {
594
604
  return
595
605
  }
596
606
 
package/index.d.ts CHANGED
@@ -3,6 +3,61 @@ import { LookupFunction } from 'net';
3
3
  import * as opentracing from "opentracing";
4
4
  import * as otel from "@opentelemetry/api";
5
5
 
6
+ declare const baseEvaluatorBrand: unique symbol
7
+ declare const baseSummaryEvaluatorBrand: unique symbol
8
+
9
+ declare const EvaluatorContext: {
10
+ new (options: {
11
+ inputData: tracer.llmobs.JSONType
12
+ outputData: tracer.llmobs.JSONType
13
+ expectedOutput?: tracer.llmobs.JSONType
14
+ metadata?: Record<string, tracer.llmobs.JSONType>
15
+ spanId?: string
16
+ traceId?: string
17
+ }): tracer.llmobs.EvaluatorContext
18
+ }
19
+
20
+ declare const SummaryEvaluatorContext: {
21
+ new (options: {
22
+ inputs: tracer.llmobs.JSONType[]
23
+ outputs: tracer.llmobs.JSONType[]
24
+ expectedOutputs: tracer.llmobs.JSONType[]
25
+ evaluationResults: Record<string, tracer.llmobs.JSONType[]>
26
+ metadata?: Array<Record<string, tracer.llmobs.JSONType>>
27
+ }): tracer.llmobs.SummaryEvaluatorContext
28
+ }
29
+
30
+ declare const EvaluatorResult: {
31
+ new (value: tracer.llmobs.JSONType, options?: tracer.llmobs.EvaluatorResultOptions): tracer.llmobs.EvaluatorResult
32
+ }
33
+
34
+ declare const MultiEvaluatorResult: {
35
+ new (values: Record<string, tracer.llmobs.JSONType | tracer.llmobs.EvaluatorResult>, prefix?: boolean): tracer.llmobs.MultiEvaluatorResult
36
+ }
37
+
38
+ declare const BaseEvaluator: {
39
+ new (name?: string): tracer.llmobs.BaseEvaluator
40
+ }
41
+
42
+ declare const BaseSummaryEvaluator: {
43
+ new (name?: string): tracer.llmobs.BaseSummaryEvaluator
44
+ }
45
+
46
+ declare const RemoteEvaluatorError: {
47
+ new (message: string, options?: {
48
+ status?: string
49
+ backendError?: Record<string, tracer.llmobs.JSONType>
50
+ }): tracer.llmobs.RemoteEvaluatorError
51
+ }
52
+
53
+ declare const RemoteEvaluator: {
54
+ new (options: {
55
+ evalName: string
56
+ transformFn?: (context: tracer.llmobs.EvaluatorContext) => Record<string, tracer.llmobs.JSONType>
57
+ }): tracer.llmobs.RemoteEvaluator
58
+ }
59
+
60
+
6
61
  /**
7
62
  * Tracer is the entry-point of the Datadog tracing implementation.
8
63
  */
@@ -162,6 +217,7 @@ interface Tracer extends opentracing.Tracer {
162
217
  *
163
218
  * @env DD_FEATURE_FLAGS_ENABLED
164
219
  * @env DD_FEATURE_FLAGS_CONFIGURATION_SOURCE
220
+ * @env DD_FLAGGING_EVALUATION_COUNTS_ENABLED
165
221
  * @beta This feature is in preview and not ready for production use
166
222
  */
167
223
  openfeature: tracer.OpenFeatureProvider;
@@ -1357,6 +1413,16 @@ declare namespace tracer {
1357
1413
  */
1358
1414
  captureTimeoutMs?: number
1359
1415
 
1416
+ /**
1417
+ * Time budget in milliseconds for evaluating a probe's condition, log message template and capture expressions.
1418
+ * An evaluation that exceeds the budget is reported as an evaluation error and the probe is not evaluated again
1419
+ * for a while.
1420
+ * @default 50
1421
+ * @env DD_DYNAMIC_INSTRUMENTATION_EVALUATION_TIMEOUT_MS
1422
+ * Programmatic configuration takes precedence over the environment variables listed above.
1423
+ */
1424
+ evaluationTimeoutMs?: number
1425
+
1360
1426
  /**
1361
1427
  * Interval in seconds between uploads of probe data.
1362
1428
  * @default 1
@@ -2137,8 +2203,9 @@ declare namespace tracer {
2137
2203
  /**
2138
2204
  * Whether to capture LLM Observability spans for this integration. When set to `false`,
2139
2205
  * the integration keeps emitting APM spans and propagating trace context, but no LLM
2140
- * Observability spans are produced. Useful when another integration already captures the
2141
- * same operation and the payloads would otherwise be stored twice.
2206
+ * Observability spans and no basic `gen_ai.*` APM tags are produced. Useful when another
2207
+ * integration already captures the same operation and the payloads would otherwise be
2208
+ * stored twice.
2142
2209
  * @default true
2143
2210
  */
2144
2211
  llmobs?: boolean;
@@ -4248,6 +4315,74 @@ declare namespace tracer {
4248
4315
 
4249
4316
  type ReadonlyJSONType = string | number | boolean | null | ReadonlyArray<ReadonlyJSONType> | { readonly [key: string]: ReadonlyJSONType }
4250
4317
 
4318
+ /** Context passed to a record-level class evaluator. */
4319
+ interface EvaluatorContext {
4320
+ inputData: JSONType
4321
+ outputData: JSONType
4322
+ expectedOutput?: JSONType
4323
+ metadata: Record<string, JSONType>
4324
+ spanId?: string
4325
+ traceId?: string
4326
+ }
4327
+
4328
+ /** Context passed to a summary class evaluator. */
4329
+ interface SummaryEvaluatorContext {
4330
+ inputs: JSONType[]
4331
+ outputs: JSONType[]
4332
+ expectedOutputs: JSONType[]
4333
+ evaluationResults: Record<string, JSONType[]>
4334
+ metadata: Array<Record<string, JSONType>>
4335
+ }
4336
+
4337
+ interface EvaluatorResultOptions {
4338
+ reasoning?: string
4339
+ assessment?: 'pass' | 'fail'
4340
+ metadata?: Record<string, JSONType>
4341
+ tags?: Record<string, JSONType>
4342
+ }
4343
+
4344
+ /** A metric value with optional evaluation details. */
4345
+ interface EvaluatorResult {
4346
+ value: JSONType
4347
+ reasoning?: string
4348
+ assessment?: 'pass' | 'fail'
4349
+ metadata?: Record<string, JSONType>
4350
+ tags?: Record<string, JSONType>
4351
+ }
4352
+
4353
+ /** A result that emits several named metrics from one evaluator invocation. */
4354
+ interface MultiEvaluatorResult {
4355
+ values: Record<string, JSONType | EvaluatorResult>
4356
+ prefix: boolean
4357
+ }
4358
+
4359
+ /** Error returned by a managed evaluator configured in Datadog. */
4360
+ interface RemoteEvaluatorError extends Error {
4361
+ status: string
4362
+ backendError: Record<string, JSONType>
4363
+ }
4364
+
4365
+ /** Evaluator that references an LLM-as-a-judge evaluator configured in Datadog. */
4366
+ interface RemoteEvaluator extends BaseEvaluator {}
4367
+ interface ExperimentSummaryEvaluation extends EvaluatorResultOptions {
4368
+ value: any
4369
+ error: string | null
4370
+ }
4371
+
4372
+ /** Base class for reusable synchronous or asynchronous record-level evaluators. */
4373
+ interface BaseEvaluator {
4374
+ readonly [baseEvaluatorBrand]: never
4375
+ name: string
4376
+ evaluate (context: EvaluatorContext): JSONType | EvaluatorResult | MultiEvaluatorResult | Promise<JSONType | EvaluatorResult | MultiEvaluatorResult>
4377
+ }
4378
+
4379
+ /** Base class for reusable synchronous or asynchronous summary evaluators. */
4380
+ interface BaseSummaryEvaluator {
4381
+ readonly [baseSummaryEvaluatorBrand]: never
4382
+ name: string
4383
+ evaluate (context: SummaryEvaluatorContext): JSONType | EvaluatorResult | MultiEvaluatorResult | Promise<JSONType | EvaluatorResult | MultiEvaluatorResult>
4384
+ }
4385
+
4251
4386
  /**
4252
4387
  * A task run over each dataset record during an experiment.
4253
4388
  */
@@ -4257,26 +4392,25 @@ declare namespace tracer {
4257
4392
  metadata?: Record<string, JSONType>
4258
4393
  ) => JSONType | Promise<JSONType>
4259
4394
 
4260
- /**
4261
- * Scores a single task output. The return type selects the metric:
4262
- * `boolean` -> boolean, `number` -> score, `string` -> categorical, anything else -> json.
4263
- */
4264
- type ExperimentEvaluator = (
4395
+ /** Scores a single task output. */
4396
+ type ExperimentEvaluatorFunction = (
4265
4397
  input: JSONType,
4266
4398
  output: JSONType,
4267
4399
  expectedOutput: JSONType
4268
- ) => JSONType | Promise<JSONType>
4400
+ ) => JSONType | EvaluatorResult | MultiEvaluatorResult | Promise<JSONType | EvaluatorResult | MultiEvaluatorResult>
4269
4401
 
4270
- /**
4271
- * Scores all rows in an experiment run and emits a summary metric.
4272
- */
4273
- type ExperimentSummaryEvaluator = (
4274
- inputs: any[],
4275
- outputs: any[],
4276
- expectedOutputs: any[],
4277
- evaluatorResults: Record<string, any[]>,
4278
- metadata?: Array<Record<string, any>>
4279
- ) => any | Promise<any>
4402
+ type ExperimentEvaluator = ExperimentEvaluatorFunction | BaseEvaluator
4403
+
4404
+ /** Scores all rows in an experiment run and emits a summary metric. */
4405
+ type ExperimentSummaryEvaluatorFunction = (
4406
+ inputs: JSONType[],
4407
+ outputs: JSONType[],
4408
+ expectedOutputs: JSONType[],
4409
+ evaluatorResults: Record<string, JSONType[]>,
4410
+ metadata?: Array<Record<string, JSONType>>
4411
+ ) => JSONType | EvaluatorResult | MultiEvaluatorResult | Promise<JSONType | EvaluatorResult | MultiEvaluatorResult>
4412
+
4413
+ type ExperimentSummaryEvaluator = ExperimentSummaryEvaluatorFunction | BaseSummaryEvaluator
4280
4414
 
4281
4415
  interface DatasetRecord {
4282
4416
  id: string | null
@@ -4307,9 +4441,9 @@ declare namespace tracer {
4307
4441
  task: ExperimentTask
4308
4442
  /** Override the configured project for this experiment. */
4309
4443
  projectName?: string
4310
- /** Evaluators keyed by metric label, or named functions. */
4444
+ /** Evaluators keyed by metric label, or named functions or class instances. */
4311
4445
  evaluators?: Record<string, ExperimentEvaluator> | ExperimentEvaluator[]
4312
- /** Summary evaluators keyed by metric label, or named functions. */
4446
+ /** Summary evaluators keyed by metric label, or named functions or class instances. */
4313
4447
  summaryEvaluators?: Record<string, ExperimentSummaryEvaluator> | ExperimentSummaryEvaluator[]
4314
4448
  description?: string
4315
4449
  config?: Record<string, JSONType>
@@ -4365,7 +4499,7 @@ declare namespace tracer {
4365
4499
  /** Whether this run had a task, row-evaluator, or summary-evaluator error. */
4366
4500
  hasError: boolean
4367
4501
  rows: ExperimentResultRow[]
4368
- summaryEvaluations: Record<string, { value: any, error: string | null }>
4502
+ summaryEvaluations: Record<string, ExperimentSummaryEvaluation>
4369
4503
  }
4370
4504
 
4371
4505
  interface ExperimentResult {
@@ -4373,7 +4507,7 @@ declare namespace tracer {
4373
4507
  /** Rows from the first run, kept as a compatibility alias. */
4374
4508
  rows: ExperimentResultRow[]
4375
4509
  /** Summary evaluator results from the first run, kept as a compatibility alias. */
4376
- summaryEvaluations: Record<string, { value: any, error: string | null }>
4510
+ summaryEvaluations: Record<string, ExperimentSummaryEvaluation>
4377
4511
  /** All experiment runs. */
4378
4512
  runs: ExperimentRun[]
4379
4513
  /** Dashboard URL for the experiment. */
@@ -4508,6 +4642,14 @@ declare namespace tracer {
4508
4642
  }
4509
4643
 
4510
4644
  interface Experiments {
4645
+ BaseEvaluator: typeof BaseEvaluator
4646
+ BaseSummaryEvaluator: typeof BaseSummaryEvaluator
4647
+ EvaluatorContext: typeof EvaluatorContext
4648
+ SummaryEvaluatorContext: typeof SummaryEvaluatorContext
4649
+ EvaluatorResult: typeof EvaluatorResult
4650
+ MultiEvaluatorResult: typeof MultiEvaluatorResult
4651
+ RemoteEvaluator: typeof RemoteEvaluator
4652
+ RemoteEvaluatorError: typeof RemoteEvaluatorError
4511
4653
  /** Create a local dataset buffer; pushed on the first experiment run. */
4512
4654
  createDataset (name: string, description?: string): Dataset
4513
4655
  createDataset (name: string, options?: CreateDatasetOptions): Dataset
@@ -4528,12 +4670,12 @@ declare namespace tracer {
4528
4670
  /**
4529
4671
  * The input content associated with the span.
4530
4672
  */
4531
- input: { content: string, role?: string }[]
4673
+ input: { content: JSONType, role?: string }[]
4532
4674
 
4533
4675
  /**
4534
4676
  * The output content associated with the span.
4535
4677
  */
4536
- output: { content: string, role?: string }[]
4678
+ output: { content: JSONType, role?: string }[]
4537
4679
 
4538
4680
  /**
4539
4681
  * Get a tag from the span.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dd-trace",
3
- "version": "5.129.0",
3
+ "version": "5.131.0",
4
4
  "description": "Datadog APM tracing client for JavaScript",
5
5
  "main": "index.js",
6
6
  "typings": "index.d.ts",
@@ -190,7 +190,7 @@
190
190
  "optionalDependencies": {
191
191
  "@datadog/libdatadog": "^0.21.0",
192
192
  "@datadog/libdatadog-extras": "^0.21.0",
193
- "@datadog/native-appsec": "11.0.2",
193
+ "@datadog/native-appsec": "11.0.3",
194
194
  "@datadog/native-iast-taint-tracking": "4.2.1",
195
195
  "@datadog/native-metrics": "4.0.0",
196
196
  "@datadog/pprof": "5.19.0",
@@ -206,7 +206,7 @@
206
206
  "@eslint/eslintrc": "^3.3.7",
207
207
  "@eslint/js": "^10.0.1",
208
208
  "@eslint/plugin-kit": "^0.7.3",
209
- "@datadog/openfeature-node-server": "2.3.0",
209
+ "@datadog/openfeature-node-server": "2.4.0",
210
210
  "@msgpack/msgpack": "^3.1.3",
211
211
  "@openfeature/core": "^1.12.0",
212
212
  "@openfeature/server-sdk": "~1.23.0",
@@ -222,14 +222,14 @@
222
222
  "bun": "1.4.2",
223
223
  "c8": "^12.0.0",
224
224
  "codeowners-audit": "^2.9.0",
225
- "eslint": "^10.10.0",
225
+ "eslint": "^10.11.0",
226
226
  "eslint-plugin-cypress": "^7.0.2",
227
227
  "eslint-plugin-import-x": "^4.16.2",
228
- "eslint-plugin-jsdoc": "^64.5.2",
228
+ "eslint-plugin-jsdoc": "^64.5.4",
229
229
  "eslint-plugin-mocha": "^12.0.2",
230
230
  "eslint-plugin-n": "^18.3.0",
231
231
  "eslint-plugin-promise": "^7.3.0",
232
- "eslint-plugin-regexp": "^3.3.0",
232
+ "eslint-plugin-regexp": "^3.3.1",
233
233
  "eslint-plugin-sonarjs": "^4.2.1",
234
234
  "eslint-plugin-unicorn": "^74.0.0",
235
235
  "express": "^5.1.0",
@@ -251,7 +251,7 @@
251
251
  "node-preload": "^0.2.1",
252
252
  "nyc": "^18.0.0",
253
253
  "octokit": "^5.0.3",
254
- "p-limit": "^7.3.2",
254
+ "p-limit": "^7.3.3",
255
255
  "proxyquire": "^2.1.3",
256
256
  "retry": "^0.13.1",
257
257
  "semifies": "^1.0.0",
@@ -165,8 +165,6 @@ ${build.initialOptions.banner.js}`
165
165
  log.warn('No git metadata available - skipping injection')
166
166
  }
167
167
 
168
- // first time is intercepted, proxy should be created, next time the original should be loaded
169
- const interceptedESMModules = new Set()
170
168
  let resolver
171
169
 
172
170
  build.onEnd(async () => {
@@ -217,11 +215,18 @@ ${build.initialOptions.banner.js}`
217
215
 
218
216
  const internal = builtins.has(args.path)
219
217
 
218
+ // Only the generated wrapper module, and the real module importing itself, must reach the real module unwrapped.
219
+ // Every other importer must get the wrapper regardless of whether it has already been loaded: onLoad runs
220
+ // concurrently with onResolve, so tracking loaded modules makes which importer sees the wrapper depend on timing,
221
+ // leaving some bindings uninstrumented. Routing a self-import to the wrapper would create a wrapper -> original ->
222
+ // wrapper cycle that can hit the wrapper's exports in their temporal dead zone.
223
+ const isInterceptedImporter = args.importer.endsWith(ESM_INTERCEPTED_SUFFIX) || args.importer === fullPathToModule
224
+
220
225
  if (args.namespace === 'file' && (
221
226
  modulesOfInterest.has(args.path) || modulesOfInterest.has(`${extracted.pkg}/${extracted.path}`))
222
227
  ) {
223
228
  // Internal module like http/fs is imported and the build output is ESM
224
- if (internal && args.kind === 'import-statement' && esmBuild && !interceptedESMModules.has(fullPathToModule)) {
229
+ if (internal && args.kind === 'import-statement' && esmBuild && !isInterceptedImporter) {
225
230
  fullPathToModule = `${INTERNAL_ESM_INTERCEPTED_PREFIX}${fullPathToModule}${ESM_INTERCEPTED_SUFFIX}`
226
231
 
227
232
  return {
@@ -263,7 +268,7 @@ ${build.initialOptions.banner.js}`
263
268
  const packageJson = JSON.parse(fs.readFileSync(/** @type {string} */(pathToPackageJson)).toString())
264
269
 
265
270
  const isESM = isESMFile(fullPathToModule, pathToPackageJson, packageJson)
266
- if (isESM && !interceptedESMModules.has(fullPathToModule)) {
271
+ if (isESM && !isInterceptedImporter) {
267
272
  fullPathToModule += ESM_INTERCEPTED_SUFFIX
268
273
  }
269
274
 
@@ -322,8 +327,6 @@ ${build.initialOptions.banner.js}`
322
327
  args.path = args.path.slice(INTERNAL_ESM_INTERCEPTED_PREFIX.length)
323
328
  }
324
329
 
325
- interceptedESMModules.add(args.path)
326
-
327
330
  resolver ??= createEsmResolver()
328
331
  const setters = await processModule({
329
332
  path: args.path,
@@ -194,6 +194,32 @@ for (const hook of getHooks('ai').values()) {
194
194
  },
195
195
  })
196
196
 
197
+ tracingChannel('orchestrion:ai:includeRuntimeContext').subscribe({
198
+ start (ctx) {
199
+ const options = ctx.arguments[0]
200
+ const runtimeContext = options?.runtimeContext
201
+ const telemetry = options?.telemetry ?? options?.experimental_telemetry
202
+
203
+ if (!runtimeContext || typeof runtimeContext !== 'object' || telemetry?.isEnabled === false) return
204
+
205
+ const keys = Object.keys(runtimeContext)
206
+ if (!keys.length) return
207
+
208
+ const includeRuntimeContext = { ...telemetry?.includeRuntimeContext }
209
+ for (const key of keys) {
210
+ if (includeRuntimeContext[key] === undefined) includeRuntimeContext[key] = true
211
+ }
212
+
213
+ ctx.arguments[0] = {
214
+ ...options,
215
+ telemetry: {
216
+ ...telemetry,
217
+ includeRuntimeContext,
218
+ },
219
+ }
220
+ },
221
+ })
222
+
197
223
  // resolveLanguageModel is called by all LLM entry points (generateText, streamText,
198
224
  // generateObject, streamObject)
199
225
  tracingChannel('orchestrion:ai:resolveLanguageModel').subscribe({
@@ -124,13 +124,48 @@ function wrapStreamIterator (iterator, ctx) {
124
124
  }
125
125
  }
126
126
 
127
+ /**
128
+ * Gives the SDK a detached copy of the response to decode, leaving the body the caller can still
129
+ * read through `asResponse()` untouched. The bytes are buffered instead of streamed from a live
130
+ * clone, and the controller is replaced, because the SDK cancels its reader and aborts its
131
+ * controller whenever a stream ends early: that abort would kill the caller's request, and on a
132
+ * cloned body that cancel blocks until the other branch is read, which the caller cannot do while
133
+ * it is still waiting for this parse.
134
+ *
135
+ * Any failure falls back to the original props and tells the wrapper to skip inspection, leaving
136
+ * the SDK to behave as it does untraced.
137
+ *
138
+ * @param {{response: Response}} props
139
+ * @param {() => void} onFailure
140
+ * @returns {object|Promise<object>}
141
+ */
142
+ function detachResponse (props, onFailure) {
143
+ const { status, statusText, headers } = props.response
144
+ try {
145
+ return props.response.clone().arrayBuffer()
146
+ .then(body => ({
147
+ ...props,
148
+ response: new Response(body, { status, statusText, headers }),
149
+ controller: new AbortController(),
150
+ }))
151
+ .catch(() => {
152
+ onFailure()
153
+ return props
154
+ })
155
+ } catch {
156
+ onFailure()
157
+ return props
158
+ }
159
+ }
160
+
127
161
  function wrapCreate (create) {
128
162
  return function (...args) {
129
163
  const stream = args[0]?.stream
130
- const preparing = !stream && messagesPrepareChannel.hasSubscribers
131
- const intercepting = !stream && messagesInterceptChannel.hasSubscribers
164
+ const tracing = anthropicTracingChannel.start.hasSubscribers
165
+ const preparing = messagesPrepareChannel.hasSubscribers
166
+ const intercepting = messagesInterceptChannel.hasSubscribers
132
167
 
133
- if (!anthropicTracingChannel.start.hasSubscribers && !preparing && !intercepting) {
168
+ if (!tracing && !preparing && !intercepting) {
134
169
  return create.apply(this, args)
135
170
  }
136
171
 
@@ -160,23 +195,60 @@ function wrapCreate (create) {
160
195
 
161
196
  let parseResult
162
197
  let wrappedResponse
198
+ // Set around a `parse()` whose body a raw-response reader also holds, so the SDK decodes a
199
+ // copy and leaves that body to the caller. Plain streaming has no second reader and copies
200
+ // nothing.
201
+ let detach = false
163
202
 
203
+ // These helpers live on each returned APIPromise, so there is no module export for
204
+ // Orchestrion to replace; wrap this instance instead.
164
205
  shimmer.wrap(apiPromise, 'parse', parse => function (...parseArgs) {
165
206
  if (parseResult) return parseResult
166
207
 
167
- parseResult = heldUntil(parse.apply(this, parseArgs), interceptCtx?.beforeResult?.())
208
+ let detachmentFailed = false
209
+ const responsePromise = detach ? this.responsePromise : undefined
210
+ if (responsePromise) {
211
+ this.responsePromise = responsePromise.then(props => detachResponse(props, () => {
212
+ detachmentFailed = true
213
+ }))
214
+ }
215
+
216
+ let result
217
+ try {
218
+ result = parse.apply(this, parseArgs)
219
+ } finally {
220
+ if (responsePromise) this.responsePromise = responsePromise
221
+ }
222
+
223
+ const onResult = interceptCtx?.onResult
224
+ parseResult = heldUntil(result, interceptCtx?.beforeResult?.())
168
225
  .then(response => {
169
226
  if (stream) {
170
- shimmer.wrap(response, Symbol.asyncIterator, iterator => wrapStreamIterator(iterator, ctx))
171
- return response
227
+ if (!onResult || detachmentFailed) {
228
+ if (tracing) {
229
+ shimmer.wrap(response, Symbol.asyncIterator, iterator => wrapStreamIterator(iterator, ctx))
230
+ }
231
+ return response
232
+ }
233
+
234
+ return Promise.resolve(onResult(response)).then(deliveredResponse => {
235
+ if (tracing) {
236
+ shimmer.wrap(
237
+ deliveredResponse,
238
+ Symbol.asyncIterator,
239
+ iterator => wrapStreamIterator(iterator, ctx)
240
+ )
241
+ }
242
+ return deliveredResponse
243
+ })
172
244
  }
173
245
 
174
- if (!interceptCtx?.onResult) {
246
+ if (!onResult) {
175
247
  finish(ctx, response)
176
248
  return response
177
249
  }
178
250
 
179
- return Promise.resolve(interceptCtx.onResult(response)).then(deliveredResponse => {
251
+ return Promise.resolve(onResult(response)).then(deliveredResponse => {
180
252
  finish(ctx, response)
181
253
  return deliveredResponse
182
254
  })
@@ -190,9 +262,26 @@ function wrapCreate (create) {
190
262
  shimmer.wrap(apiPromise, 'asResponse', origAsResponse => function (...asResponseArgs) {
191
263
  return heldUntil(origAsResponse.apply(this, asResponseArgs), interceptCtx?.beforeResult?.())
192
264
  .then(response => {
265
+ if (stream) {
266
+ // A caller holding the parsed stream closes the span as it drains it. When only
267
+ // the raw body is read, decode the stream here so it is still evaluated.
268
+ if (!interceptCtx?.onResult) return response
269
+ if (parseResult) return parseResult.then(() => response)
270
+
271
+ detach = true
272
+ try {
273
+ return apiPromise.parse().then(() => {
274
+ finish(ctx)
275
+ return response
276
+ })
277
+ } finally {
278
+ detach = false
279
+ }
280
+ }
281
+
193
282
  // Wrap json()/text()/clone() so the span still closes on the raw-response path,
194
283
  // and not twice for the same response.
195
- if (!stream && wrappedResponse !== response) {
284
+ if (wrappedResponse !== response) {
196
285
  wrappedResponse = response
197
286
  wrapRawResponse(response, ctx, interceptCtx)
198
287
  }
@@ -202,6 +291,19 @@ function wrapCreate (create) {
202
291
  })
203
292
  }
204
293
 
294
+ // `withResponse()` is the one SDK call that hands out the raw response *and* the parsed
295
+ // stream; it invokes `parse()` synchronously, so the flag only has to outlive that call.
296
+ if (stream && interceptCtx?.onResult && typeof apiPromise.withResponse === 'function') {
297
+ shimmer.wrap(apiPromise, 'withResponse', withResponse => function (...withResponseArgs) {
298
+ detach = true
299
+ try {
300
+ return withResponse.apply(this, withResponseArgs)
301
+ } finally {
302
+ detach = false
303
+ }
304
+ })
305
+ }
306
+
205
307
  anthropicTracingChannel.end.publish(ctx)
206
308
 
207
309
  return apiPromise