dd-trace 6.18.0 → 6.20.0

package/packages/dd-trace/src/llmobs/experiments/experiment.js

@@ -2,7 +2,17 @@
 
 const id = require('../../id')
 const log = require('../../log')
+const { validateAssessment, validateReasoning } = require('../eval-metric')
 
+const {
+  BaseEvaluator,
+  BaseSummaryEvaluator,
+  EvaluatorContext,
+  EvaluatorResult,
+  MultiEvaluatorResult,
+  SummaryEvaluatorContext,
+} = require('./evaluator')
+const { RemoteEvaluator, RemoteEvaluatorError } = require('./remote-evaluator')
 const { Row, ExperimentResult, ExperimentRun } = require('./result')
 const {
   buildSpanMetadata,
@@ -69,7 +79,17 @@ function toSpan (row, metadata, ids, spanName, userTags, recordTags) {
 
 // One metric per evaluator per row or summary evaluator.
 function toMetric (
-  label, value, errorMessage, spanId, traceId, timestampMs, experimentId, userTags, source = 'custom', ids = {}
+  label,
+  value,
+  errorMessage,
+  spanId,
+  traceId,
+  timestampMs,
+  experimentId,
+  userTags,
+  source = 'custom',
+  ids = {},
+  extras = {}
 ) {
   const metric = {
     metric_source: source,
@@ -77,17 +97,30 @@ function toMetric (
     span_id: spanId,
     trace_id: traceId,
     timestamp_ms: timestampMs,
-    tags: buildTags(userTags, {
+    tags: buildTags({ ...userTags, ...extras.tags }, {
       experiment_id: experimentId,
       run_id: ids.runId,
       run_iteration: ids.runIteration,
+      project_name: userTags.project_name,
     }),
     experiment_id: experimentId,
   }
 
+  if (extras.reasoning !== undefined) {
+    validateReasoning(extras.reasoning)
+    metric.reasoning = extras.reasoning
+  }
+  if (extras.assessment !== undefined) {
+    validateAssessment(extras.assessment)
+    metric.assessment = extras.assessment
+  }
+  if (extras.metadata !== undefined) metric.metadata = extras.metadata
+  if (extras.status !== undefined) metric.status = extras.status
+  if (extras.evalSourceType !== undefined) metric.eval_source_type = extras.evalSourceType
+
   if (errorMessage !== null) {
     metric.metric_type = 'categorical'
-    metric.error = { message: errorMessage }
+    metric.error = extras.error ?? { message: errorMessage }
     return metric
   }
 
@@ -100,6 +133,31 @@ function toMetric (
   return metric
 }
 
+/**
+ * @param {unknown} result
+ * @param {string} label
+ * @returns {Array<{label: string, value: unknown, extras: object}>}
+ */
+function extractEvaluatorResults (result, label) {
+  if (!(result instanceof MultiEvaluatorResult)) {
+    return [{
+      label,
+      value: result instanceof EvaluatorResult ? result.value : result,
+      extras: result instanceof EvaluatorResult ? result : {},
+    }]
+  }
+
+  const results = []
+  for (const [key, value] of Object.entries(result.values)) {
+    results.push({
+      label: result.prefix ? `${label}-${key}` : key,
+      value: value instanceof EvaluatorResult ? value.value : value,
+      extras: value instanceof EvaluatorResult ? value : {},
+    })
+  }
+  return results
+}
+
 function createFallbackSpanContext (startNs) {
   // Root-span id generation, same convention as opentracing/span.js: a single
   // random 64-bit id for the span, reused as the trace id's low 64 bits with a
@@ -543,13 +601,15 @@ class Experiment {
     const metrics = []
     const evaluatorResults = {}
     let hasRowError = false
-    for (const [label] of this.#evaluators) evaluatorResults[label] = []
 
     for (let i = 0; i < results.length; i++) {
       const result = results[i]
       const row = result.row
       rows[i] = row
-      for (const [label] of this.#evaluators) {
+      for (const label of Object.keys(result.evaluatorValues)) {
+        if (!Object.hasOwn(evaluatorResults, label)) evaluatorResults[label] = new Array(i).fill(null)
+      }
+      for (const label of Object.keys(evaluatorResults)) {
         const value = Object.hasOwn(result.evaluatorValues, label) ? result.evaluatorValues[label] : null
         evaluatorResults[label].push(value)
       }
@@ -671,8 +731,13 @@ class Experiment {
 
     const evaluatorResults = await Promise.all(pending)
     for (const result of evaluatorResults) {
-      if (result.metric !== null) metrics.push(result.metric)
-      evaluatorValues[result.label] = result.value
+      for (const value of result.values) {
+        if (Object.hasOwn(evaluatorValues, value.label)) {
+          throw new Error(`Evaluator metric label '${value.label}' was emitted more than once`)
+        }
+        if (value.metric !== null) metrics.push(value.metric)
+        evaluatorValues[value.label] = value.value
+      }
       if (result.error !== undefined && firstError === undefined) firstError = result.error
     }
     if (firstError !== undefined) throw firstError
@@ -703,64 +768,101 @@ class Experiment {
       row.evaluationErrors[label] = TASK_ERROR_MESSAGE
       return {
         label,
-        value: null,
-        metric: toMetric(
+        values: [{
           label,
-          null,
-          TASK_ERROR_MESSAGE,
-          row.spanId,
-          row.traceId,
-          timestampMs,
-          experimentId,
-          this.#tags,
-          'custom',
-          { runId, runIteration }
-        ),
+          value: null,
+          metric: toMetric(
+            label,
+            null,
+            TASK_ERROR_MESSAGE,
+            row.spanId,
+            row.traceId,
+            timestampMs,
+            experimentId,
+            this.#tags,
+            'custom',
+            { runId, runIteration }
+          ),
+        }],
+        error: undefined,
       }
     }
 
+    const isRemoteEvaluator = evaluator instanceof RemoteEvaluator
     try {
-      const value = await limit(() => this.#runWithRetries(
-        () => evaluator(record.input, row.output, record.expectedOutput),
-        maxRetries,
-        retryDelay
-      ), throwOnErrors)
-      row.evaluations[label] = value
-      return {
-        label,
-        value,
-        metric: toMetric(
-          label,
-          value,
-          null,
-          row.spanId,
-          row.traceId,
-          timestampMs,
-          experimentId,
-          this.#tags,
-          'custom',
-          { runId, runIteration }
-        ),
+      let evaluate
+      if (evaluator instanceof BaseEvaluator) {
+        const context = new EvaluatorContext({
+          inputData: record.input,
+          outputData: row.output,
+          expectedOutput: record.expectedOutput,
+          metadata: buildSpanMetadata(record.metadata, this.#config),
+          spanId: row.spanId,
+          traceId: row.traceId,
+        })
+        evaluate = isRemoteEvaluator
+          ? () => evaluator.evaluate(context, this.#client)
+          : () => evaluator.evaluate(context)
+      } else {
+        evaluate = () => evaluator(record.input, row.output, record.expectedOutput)
+      }
+      const result = await limit(() => this.#runWithRetries(evaluate, maxRetries, retryDelay), throwOnErrors)
+      const values = []
+      for (const evaluatorResult of extractEvaluatorResults(result, label)) {
+        row.evaluations[evaluatorResult.label] = evaluatorResult.value
+        values.push({
+          label: evaluatorResult.label,
+          value: evaluatorResult.value,
+          metric: toMetric(
+            evaluatorResult.label,
+            evaluatorResult.value,
+            null,
+            row.spanId,
+            row.traceId,
+            timestampMs,
+            experimentId,
+            this.#tags,
+            'custom',
+            { runId, runIteration },
+            evaluatorResult.extras
+          ),
+        })
       }
+      return { label, values, error: undefined }
     } catch (err) {
       if (throwOnErrors) throw err
-      const msg = err.message ?? String(err)
+      const backendError = err instanceof RemoteEvaluatorError && hasEntries(err.backendError)
+        ? err.backendError
+        : undefined
+      const msg = backendError?.message ?? err.message ?? String(err)
       row.evaluationErrors[label] = msg
+      const extras = isRemoteEvaluator
+        ? {
+            error: backendError,
+            status: err.status ?? 'ERROR',
+            evalSourceType: 'managed',
+          }
+        : {}
       return {
         label,
-        value: null,
-        metric: toMetric(
+        values: [{
           label,
-          null,
-          msg,
-          row.spanId,
-          row.traceId,
-          timestampMs,
-          experimentId,
-          this.#tags,
-          'custom',
-          { runId, runIteration }
-        ),
+          value: null,
+          metric: toMetric(
+            label,
+            null,
+            msg,
+            row.spanId,
+            row.traceId,
+            timestampMs,
+            experimentId,
+            this.#tags,
+            'custom',
+            { runId, runIteration },
+            extras
+          ),
+        }],
+        error: undefined,
       }
     }
   }
@@ -892,7 +994,6 @@ class Experiment {
     const summaryEvaluations = {}
     const timestampMs = Date.now()
     const pending = new Array(this.#summaryEvaluators.length)
-    let firstError
 
     for (let i = 0; i < this.#summaryEvaluators.length; i++) {
       const [label, evaluator] = this.#summaryEvaluators[i]
@@ -917,14 +1018,14 @@ class Experiment {
       throw err
     }
     for (const result of results) {
-      if (result.error !== undefined) {
-        if (firstError === undefined) firstError = result.error
-        continue
+      for (const value of result.values) {
+        if (Object.hasOwn(summaryEvaluations, value.label)) {
+          throw new Error(`Summary evaluator metric label '${value.label}' was emitted more than once`)
+        }
+        summaryEvaluations[value.label] = value.evaluation
+        options.metrics.push(value.metric)
       }
-      summaryEvaluations[result.label] = result.evaluation
-      options.metrics.push(result.metric)
     }
-    if (firstError !== undefined) throw firstError
 
     return summaryEvaluations
   }
@@ -941,45 +1042,67 @@ class Experiment {
     options,
   }) {
     try {
-      const value = await options.limit(() => this.#runWithRetries(
-        () => evaluator(inputs, outputs, expectedOutputs, evaluatorResults, metadata),
+      const context = new SummaryEvaluatorContext({
+        inputs,
+        outputs,
+        expectedOutputs,
+        evaluationResults: evaluatorResults,
+        metadata,
+      })
+      const evaluate = evaluator instanceof BaseSummaryEvaluator
+        ? () => evaluator.evaluate(context)
+        : () => evaluator(inputs, outputs, expectedOutputs, evaluatorResults, metadata)
+      const result = await options.limit(() => this.#runWithRetries(
+        evaluate,
         options.maxRetries,
         options.retryDelay
       ), options.throwOnErrors)
-      return {
-        label,
-        evaluation: { value, error: null },
-        metric: toMetric(
-          label,
-          value,
-          null,
-          '',
-          '',
-          timestampMs,
-          options.experimentId,
-          this.#tags,
-          'summary',
-          { runId: options.runId, runIteration: options.runIteration }
-        ),
+      const values = []
+      for (const evaluatorResult of extractEvaluatorResults(result, label)) {
+        const evaluation = { value: evaluatorResult.value, error: null }
+        if (evaluatorResult.extras.reasoning !== undefined) evaluation.reasoning = evaluatorResult.extras.reasoning
+        if (evaluatorResult.extras.assessment !== undefined) evaluation.assessment = evaluatorResult.extras.assessment
+        if (evaluatorResult.extras.metadata !== undefined) evaluation.metadata = evaluatorResult.extras.metadata
+        if (evaluatorResult.extras.tags !== undefined) evaluation.tags = evaluatorResult.extras.tags
+        values.push({
+          label: evaluatorResult.label,
+          evaluation,
+          metric: toMetric(
+            evaluatorResult.label,
+            evaluatorResult.value,
+            null,
+            '',
+            '',
+            timestampMs,
+            options.experimentId,
+            this.#tags,
+            'summary',
+            { runId: options.runId, runIteration: options.runIteration },
+            evaluatorResult.extras
+          ),
+        })
       }
+      return { values }
     } catch (err) {
       if (options.throwOnErrors) throw err
       const msg = err.message ?? String(err)
       return {
-        label,
-        evaluation: { value: null, error: msg },
-        metric: toMetric(
+        values: [{
           label,
-          null,
-          msg,
-          '',
-          '',
-          timestampMs,
-          options.experimentId,
-          this.#tags,
-          'summary',
-          { runId: options.runId, runIteration: options.runIteration }
-        ),
+          evaluation: { value: null, error: msg },
+          metric: toMetric(
+            label,
+            null,
+            msg,
+            '',
+            '',
+            timestampMs,
+            options.experimentId,
+            this.#tags,
+            'summary',
+            { runId: options.runId, runIteration: options.runIteration }
+          ),
+        }],
       }
     }
   }

package/packages/dd-trace/src/llmobs/experiments/index.js

@@ -4,6 +4,8 @@ const log = require('../../log')
 const { ExperimentsClient } = require('./client')
 const { Dataset } = require('./dataset')
 const { Experiment, ExternalExperiment } = require('./experiment')
+const evaluatorTypes = require('./evaluator')
+const remoteEvaluatorTypes = require('./remote-evaluator')
 const { validateTagsList } = require('./util')
 const NoopExperiments = require('./noop')
 
@@ -40,6 +42,38 @@ class Experiments {
     this.#client = this.#clientForProject(this.#projectName)
   }
 
+  get BaseEvaluator () {
+    return evaluatorTypes.BaseEvaluator
+  }
+
+  get BaseSummaryEvaluator () {
+    return evaluatorTypes.BaseSummaryEvaluator
+  }
+
+  get EvaluatorContext () {
+    return evaluatorTypes.EvaluatorContext
+  }
+
+  get SummaryEvaluatorContext () {
+    return evaluatorTypes.SummaryEvaluatorContext
+  }
+
+  get EvaluatorResult () {
+    return evaluatorTypes.EvaluatorResult
+  }
+
+  get MultiEvaluatorResult () {
+    return evaluatorTypes.MultiEvaluatorResult
+  }
+
+  get RemoteEvaluator () {
+    return remoteEvaluatorTypes.RemoteEvaluator
+  }
+
+  get RemoteEvaluatorError () {
+    return remoteEvaluatorTypes.RemoteEvaluatorError
+  }
+
   /**
    * @param {string} projectName
    * @returns {ExperimentsClient}
@@ -220,4 +254,9 @@ function createExperiments (config, llmobs) {
   return new Experiments(config, llmobs)
 }
 
-module.exports = { Experiments, createExperiments }
+module.exports = {
+  Experiments,
+  createExperiments,
+  ...evaluatorTypes,
+  ...remoteEvaluatorTypes,
+}

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -1,7 +1,25 @@
 'use strict'
 
 const log = require('../../log')
-const { ExternalExperiment } = require('./experiment')
+
+let ExternalExperiment
+let evaluatorTypes
+let remoteEvaluatorTypes
+
+function getExternalExperiment () {
+  ExternalExperiment ??= require('./experiment').ExternalExperiment
+  return ExternalExperiment
+}
+
+function getEvaluatorTypes () {
+  evaluatorTypes ??= require('./evaluator')
+  return evaluatorTypes
+}
+
+function getRemoteEvaluatorTypes () {
+  remoteEvaluatorTypes ??= require('./remote-evaluator')
+  return remoteEvaluatorTypes
+}
 
 const NOOP_EXPERIMENT_ID = '00000000-0000-0000-0000-000000000000'
 const NOOP_SPAN_ID = '0000000000000000'
@@ -246,6 +264,38 @@ class NoopExperiments {
     log.warn('LLMObs experiments unavailable: %s', this.#reason)
   }
 
+  get BaseEvaluator () {
+    return getEvaluatorTypes().BaseEvaluator
+  }
+
+  get BaseSummaryEvaluator () {
+    return getEvaluatorTypes().BaseSummaryEvaluator
+  }
+
+  get EvaluatorContext () {
+    return getEvaluatorTypes().EvaluatorContext
+  }
+
+  get SummaryEvaluatorContext () {
+    return getEvaluatorTypes().SummaryEvaluatorContext
+  }
+
+  get EvaluatorResult () {
+    return getEvaluatorTypes().EvaluatorResult
+  }
+
+  get MultiEvaluatorResult () {
+    return getEvaluatorTypes().MultiEvaluatorResult
+  }
+
+  get RemoteEvaluator () {
+    return getRemoteEvaluatorTypes().RemoteEvaluator
+  }
+
+  get RemoteEvaluatorError () {
+    return getRemoteEvaluatorTypes().RemoteEvaluatorError
+  }
+
   createDataset (name, options = {}) {
     this.#warn()
     return new NoopDataset(name, options)
@@ -267,7 +317,8 @@ class NoopExperiments {
    */
   startExperiment (options = {}) {
     this.#warn()
-    return Promise.resolve(new ExternalExperiment(new NoopExperiment(options.name, true)))
+    const NoopExternalExperiment = getExternalExperiment()
+    return Promise.resolve(new NoopExternalExperiment(new NoopExperiment(options.name, true)))
   }
 }
 

package/packages/dd-trace/src/llmobs/experiments/remote-evaluator.js

@@ -0,0 +1,110 @@
+'use strict'
+
+const { BaseEvaluator, EvaluatorResult } = require('./evaluator')
+const { hasEntries } = require('./util')
+
+/**
+ * @typedef {{value?: unknown, reasoning?: string, assessment?: string, status?: string}} RemoteEvaluatorResponse
+ * @typedef {{evaluatorInfer: (evalName: string, context: object) => Promise<RemoteEvaluatorResponse>}}
+ *   RemoteEvaluatorClient
+ */
+
+/**
+ * @param {import('./evaluator').EvaluatorContext} context
+ */
+function defaultContextTransform (context) {
+  const transformed = {
+    span_input: context.inputData,
+    span_output: context.outputData,
+  }
+
+  const meta = {}
+  let hasMeta = false
+  if (context.expectedOutput != null) {
+    meta.expected_output = context.expectedOutput
+    hasMeta = true
+  }
+  if (hasEntries(context.metadata)) {
+    meta.metadata = context.metadata
+    hasMeta = true
+  }
+  if (hasMeta) transformed.meta = meta
+
+  if (context.spanId) transformed.span_id = context.spanId
+  if (context.traceId) transformed.trace_id = context.traceId
+
+  return transformed
+}
+
+class RemoteEvaluatorResult extends EvaluatorResult {
+  /**
+   * @param {unknown} value
+   * @param {{reasoning?: string, assessment?: string, status?: string}} [options]
+   */
+  constructor (value, { reasoning, assessment, status } = {}) {
+    super(value, { reasoning, assessment })
+    this.status = status
+    this.evalSourceType = 'managed'
+  }
+}
+
+class RemoteEvaluatorError extends Error {
+  /**
+   * @param {string} message
+   * @param {{status?: string, backendError?: object}} [options]
+   */
+  constructor (message, { status = 'ERROR', backendError = {} } = {}) {
+    super(message)
+    this.name = 'RemoteEvaluatorError'
+    this.status = status
+    this.backendError = backendError
+  }
+}
+
+/**
+ * Evaluator that references an LLM-as-a-judge evaluator configured in Datadog.
+ */
+class RemoteEvaluator extends BaseEvaluator {
+  #evalName
+  #transformFn
+
+  /**
+   * @param {{evalName: string, transformFn?: (context: import('./evaluator').EvaluatorContext) => object}} options
+   */
+  constructor (options) {
+    super()
+    const { evalName, transformFn } = options ?? {}
+    if (typeof evalName !== 'string' || evalName.trim() === '') {
+      throw new TypeError('evalName must be a non-empty string')
+    }
+    if (transformFn !== undefined && typeof transformFn !== 'function') {
+      throw new TypeError('transformFn must be a function')
+    }
+
+    this.name = evalName.trim()
+    this.#evalName = this.name
+    this.#transformFn = transformFn ?? defaultContextTransform
+  }
+
+  /**
+   * @param {import('./evaluator').EvaluatorContext} context
+   * @param {RemoteEvaluatorClient} [client]
+   */
+  evaluate (context, client) {
+    if (client === undefined || typeof client.evaluatorInfer !== 'function') {
+      throw new Error('RemoteEvaluator can only be evaluated as part of an experiment')
+    }
+
+    const transformed = this.#transformFn(context)
+    return client.evaluatorInfer(this.#evalName, transformed).then(result => new RemoteEvaluatorResult(
+      result?.value,
+      {
+        reasoning: result?.reasoning,
+        assessment: result?.assessment,
+        status: result?.status,
+      }
+    ))
+  }
+}
+
+module.exports = { RemoteEvaluator, RemoteEvaluatorError }