dd-trace 6.10.0 → 6.12.0

package/packages/dd-trace/src/llmobs/experiments/experiment.js

@@ -1,24 +1,27 @@
 'use strict'
 
 const id = require('../../id')
+const log = require('../../log')
 
-const { API_BASE_PATH } = require('./client')
 const { Row, ExperimentResult, ExperimentRun } = require('./result')
 const {
-  buildExperimentTagObject,
   buildSpanMetadata,
   buildTags,
+  durationNs,
   hasEntries,
   inferMetricType,
   normalizeEvaluators,
+  mergeTags,
   normalizeJsonMetricValue,
+  recordTagsToObject,
   sleep,
   stringify,
+  timestampMs,
   validateEvaluatorName,
 } = require('./util')
 
 // One span per experiment row (LLM Obs experiment span wire format).
-function toSpan (row, metadata, ids, spanName, userTags) {
+function toSpan (row, metadata, ids, spanName, userTags, recordTags) {
   const meta = {
     input: row.input ?? null,
     output: row.output ?? null,
@@ -28,9 +31,23 @@ function toSpan (row, metadata, ids, spanName, userTags) {
     meta.metadata = metadata
   }
   if (row.isError) {
-    meta.error = { type: row.errorType ?? '', message: row.errorMessage ?? '', stack: '' }
+    meta.error = { type: row.errorType ?? '', message: row.errorMessage ?? '', stack: row.errorStack ?? '' }
   }
 
+  const tags = buildTags({
+    ...userTags,
+    ...recordTagsToObject(recordTags),
+  }, {
+    experiment_id: ids.experimentId,
+    run_id: ids.runId,
+    run_iteration: ids.runIteration,
+    project_id: ids.projectId,
+    dataset_id: ids.datasetId,
+    dataset_record_id: ids.datasetRecordId,
+    dataset_name: ids.datasetName,
+    experiment_name: ids.experimentName,
+  })
+
   return {
     span_id: row.spanId,
     trace_id: row.traceId,
@@ -41,13 +58,7 @@ function toSpan (row, metadata, ids, spanName, userTags) {
     duration: row.durationNs,
     status: row.isError ? 'error' : 'ok',
     meta,
-    tags: buildTags(userTags, {
-      experiment_id: ids.experimentId,
-      run_id: ids.runId,
-      run_iteration: ids.runIteration,
-      dataset_id: ids.datasetId,
-      dataset_record_id: ids.datasetRecordId,
-    }),
+    tags,
   }
 }
 
@@ -91,11 +102,34 @@ function createFallbackSpanContext (startNs) {
   return { spanId, traceId }
 }
 
+function normalizeError (error) {
+  if (error == null) {
+    return { errorType: null, errorMessage: null, errorStack: '' }
+  }
+
+  if (typeof error === 'string') {
+    return { errorType: 'Error', errorMessage: error, errorStack: '' }
+  }
+
+  return {
+    errorType: error.type ?? error.name ?? 'Error',
+    errorMessage: error.message ?? String(error),
+    errorStack: error.stack ?? '',
+  }
+}
+
+function errorMessage (error) {
+  if (error == null) return null
+  if (typeof error === 'string') return error
+  return error.message ?? String(error)
+}
+
 // Builder + run() orchestration: runs rows sequentially, emits one root span
 // per dataset row, and posts spans + metrics to the experiments events API.
 class Experiment {
   #client
   #llmobs
+  #external
   #name
   #description
   #dataset
@@ -104,24 +138,37 @@ class Experiment {
   #summaryEvaluators
   #config
   #tags
+  #metadata
+  #projectId
   #experimentId
+  #runId
+  #runIteration
 
   constructor (client, options = {}, llmobs) {
     if (!options.name) throw new Error('Experiment name is required')
-    if (!options.dataset) throw new Error('Experiment dataset is required')
-    if (typeof options.task !== 'function') throw new Error('Experiment task is required')
+    this.#external = options.external === true
+    if (!this.#external) {
+      if (!options.dataset) throw new Error('Experiment dataset is required')
+      if (typeof options.task !== 'function') throw new Error('Experiment task is required')
+    }
 
     this.#client = client
     this.#llmobs = llmobs
     this.#name = options.name
     this.#description = options.description ?? ''
-    this.#dataset = options.dataset
+    this.#dataset = options.dataset ?? {}
     this.#task = options.task
     this.#evaluators = normalizeEvaluators(options.evaluators, 'row')
     this.#summaryEvaluators = normalizeEvaluators(options.summaryEvaluators, 'summary')
     this.#config = { ...options.config }
+    const filterTags = this.#dataset.filterTags?.() ?? []
+    if (filterTags.length > 0) this.#config.filtered_record_tags = filterTags
     this.#tags = { ...options.tags }
+    this.#metadata = { ...options.metadata }
+    this.#projectId = null
     this.#experimentId = null
+    this.#runId = null
+    this.#runIteration = null
   }
 
   name () {
@@ -137,7 +184,177 @@ class Experiment {
     return `${this.#client.appBase}/llm/experiments/${this.#experimentId}`
   }
 
+  /**
+   * @returns {Promise<Experiment>}
+   */
+  async start () {
+    if (!this.#external) throw new Error('Experiment is not externally driven')
+
+    this.#projectId = await this.#client.ensureProjectId()
+
+    const dataset = await this.#ensureExternalDataset()
+    const attributes = {
+      name: this.#name,
+      project_id: this.#projectId,
+      dataset_id: dataset.id,
+      description: this.#description,
+      ensure_unique: true,
+      run_count: 1,
+    }
+    if (dataset.version != null) attributes.dataset_version = dataset.version
+    if (hasEntries(this.#config)) attributes.config = this.#config
+
+    const metadata = { ...this.#metadata }
+    if (hasEntries(this.#tags)) metadata.tags = buildTags(this.#tags, {})
+    if (hasEntries(metadata)) attributes.metadata = metadata
+
+    let created
+    try {
+      created = await this.#client.createExperiment(attributes)
+    } catch (err) {
+      throw new Error(`Failed to create experiment '${this.#name}': ${err.message}`)
+    }
+    this.#experimentId = created.experimentId
+    this.#runId = id().toString(16).padStart(16, '0')
+    this.#runIteration = 0
+    return this
+  }
+
+  async #ensureExternalDataset () {
+    if (this.#dataset.id) {
+      return { id: this.#dataset.id, version: this.#dataset.version }
+    }
+
+    const name = this.#dataset.name ?? `${this.#name} dataset`
+    const description = this.#dataset.description ??
+      `Placeholder dataset for externally-driven experiment '${this.#name}'`
+    let response
+    try {
+      response = await this.#client.createDataset(this.#projectId, { name, description })
+    } catch (err) {
+      throw new Error(`Failed to create placeholder dataset '${name}': ${err.message}`)
+    }
+
+    const id = response.id()
+    if (id === null) {
+      throw new Error(`Failed to create placeholder dataset '${name}': backend response is missing dataset id`)
+    }
+
+    this.#dataset = {
+      ...this.#dataset,
+      id,
+      version: this.#dataset.version ?? response.version(),
+    }
+    return { id: this.#dataset.id, version: this.#dataset.version }
+  }
+
+  /**
+   * @param {object} input
+   * @returns {Promise<{experimentId: string, spanId: string, traceId: string, url: string | null}>}
+   */
+  async submitSpan (input = {}) {
+    if (this.#experimentId === null || this.#projectId === null) {
+      throw new Error('Experiment has not been started')
+    }
+
+    const startMs = timestampMs(input.startedAt)
+    const startNs = Math.round(startMs * 1e6)
+    const { spanId, traceId } = createFallbackSpanContext(startNs)
+    const error = normalizeError(input.error)
+    const row = new Row({
+      spanId,
+      traceId,
+      startNs,
+      durationNs: durationNs(input, startMs),
+      input: input.input,
+      output: input.output,
+      expectedOutput: input.expectedOutput,
+      errorType: error.errorType,
+      errorMessage: error.errorMessage,
+      errorStack: error.errorStack,
+      evaluations: {},
+      evaluationErrors: {},
+    })
+
+    const span = toSpan(row, input.metadata, {
+      experimentId: this.#experimentId,
+      projectId: this.#projectId,
+      datasetId: this.#dataset.id,
+      datasetRecordId: input.datasetRecordId,
+      runId: input.runId ?? this.#runId,
+      runIteration: input.runIteration ?? this.#runIteration,
+    }, input.name ?? this.#name, mergeTags(this.#tags, input.tags))
+
+    await this.#postEvents(this.#experimentId, [span], [])
+
+    return {
+      experimentId: this.#experimentId,
+      spanId,
+      traceId,
+      url: this.url(),
+    }
+  }
+
+  /**
+   * @param {{experimentId?: string, spanId: string, traceId?: string}} span
+   * @param {object[]} metrics
+   * @returns {Promise<void>}
+   */
+  async submitEvaluationMetrics (span, metrics) {
+    if (this.#experimentId === null) {
+      throw new Error('Experiment has not been started')
+    }
+    if (!span?.spanId) {
+      throw new Error('Experiment span id is required')
+    }
+
+    const experimentId = span.experimentId ?? this.#experimentId
+    if (experimentId !== this.#experimentId) {
+      throw new Error(`Experiment span belongs to '${experimentId}', not '${this.#experimentId}'`)
+    }
+    if (!span.traceId) {
+      throw new Error('Experiment trace id is required')
+    }
+
+    const payload = []
+    for (const metric of metrics) {
+      validateEvaluatorName(metric.label)
+      const metricError = errorMessage(metric.error)
+      if (metric.value === undefined && metricError == null) {
+        log.warn('LLMObs experiments: skipping external metric %s because it has neither value nor error', metric.label)
+        continue
+      }
+      payload.push(toMetric(
+        metric.label,
+        metric.value,
+        metricError,
+        span.spanId,
+        span.traceId,
+        timestampMs(metric.timestamp),
+        experimentId,
+        mergeTags(this.#tags, metric.tags),
+        metric.source ?? 'custom'
+      ))
+    }
+
+    if (payload.length === 0) return
+    await this.#postEvents(experimentId, [], payload)
+  }
+
+  /**
+   * @param {object} options
+   * @returns {Promise<void>}
+   */
+  async close (options = {}) {
+    if (this.#experimentId === null) return
+    const error = errorMessage(options.error)
+    const status = options.status ?? (error === null ? 'completed' : 'failed')
+    await this.#updateStatus(this.#experimentId, status, error, false)
+  }
+
   async run (options = {}) {
+    if (this.#external) throw new Error('Externally-driven experiments cannot run local tasks')
+
     const {
       maxRetries = 0,
       retryDelay = (attempt) => 100 * (attempt + 1),
@@ -168,17 +385,16 @@ class Experiment {
     }
     const datasetVersion = this.#dataset.version()
     if (datasetVersion !== null) attributes.dataset_version = datasetVersion
+    // eslint-disable-next-line no-restricted-syntax -- faster than tracking entries while copying arbitrary config
     if (Object.keys(this.#config).length > 0) attributes.config = this.#config
 
     let created
     try {
-      created = await this.#client.request('POST', `${API_BASE_PATH}/experiments`, {
-        data: { type: 'experiments', attributes },
-      })
+      created = await this.#client.createExperiment(attributes)
     } catch (err) {
       throw new Error(`Failed to create experiment '${this.#name}': ${err.message}`)
     }
-    this.#experimentId = created?.data?.id ?? null
+    this.#experimentId = created.experimentId
     const experimentId = this.#experimentId
     const runId = id().toString(16).padStart(16, '0')
     const runIteration = 0
@@ -249,9 +465,11 @@ class Experiment {
             projectId,
             datasetId,
             datasetRecordId,
+            datasetName: this.#dataset.name(),
+            experimentName: this.#name,
             runId,
             runIteration,
-          }, this.#task.name || this.#name, this.#tags))
+          }, this.#task.name || this.#name, this.#tags, record.tags))
         }
       }
 
@@ -316,7 +534,7 @@ class Experiment {
       dataset_name: this.#dataset.name(),
       experiment_name: this.#name,
     }
-    const tags = buildExperimentTagObject(this.#tags, autoTags)
+    const tags = mergeTags(this.#tags, { ...recordTagsToObject(record.tags), ...autoTags })
 
     const execute = () => this.#runWithRetries(
       () => this.#task(record.input, this.#config, record.metadata),
@@ -454,24 +672,79 @@ class Experiment {
   async #postEvents (experimentId, spans, metrics) {
     const attributes = { metrics }
     if (spans.length > 0) attributes.spans = spans
-    await this.#client.request('POST', `${API_BASE_PATH}/experiments/${experimentId}/events`, {
-      data: { type: 'experiments', attributes },
-    })
+    await this.#client.postExperimentEvents(experimentId, attributes)
   }
 
-  async #updateStatus (experimentId, status, error) {
+  async #updateStatus (experimentId, status, error, suppressErrors = true) {
     if (!experimentId) return
     // The experiment-update model has no status field, so this is a direct PATCH.
     const attributes = { status }
     if (error !== null) attributes.error = error
     try {
-      await this.#client.request('PATCH', `${API_BASE_PATH}/experiments/${experimentId}`, {
-        data: { type: 'experiments', attributes },
-      })
-    } catch {
+      await this.#client.updateExperiment(experimentId, attributes)
+    } catch (err) {
+      if (!suppressErrors) throw err
       // Status update is best-effort; never let it mask the real result/error.
     }
   }
 }
 
-module.exports = { Experiment, normalizeEvaluators, validateEvaluatorName }
+class ExternalExperiment {
+  #experiment
+
+  /**
+   * @param {Pick<Experiment, 'name' | 'experimentId' | 'url' | 'submitSpan' | 'submitEvaluationMetrics' |
+   *   'close'>} experiment
+   */
+  constructor (experiment) {
+    this.#experiment = experiment
+  }
+
+  /**
+   * @returns {string}
+   */
+  name () {
+    return this.#experiment.name()
+  }
+
+  /**
+   * @returns {string | null}
+   */
+  experimentId () {
+    return this.#experiment.experimentId()
+  }
+
+  /**
+   * @returns {string | null}
+   */
+  url () {
+    return this.#experiment.url()
+  }
+
+  /**
+   * @param {object} [input]
+   * @returns {Promise<object>}
+   */
+  submitSpan (input) {
+    return this.#experiment.submitSpan(input)
+  }
+
+  /**
+   * @param {object} span
+   * @param {object[]} metrics
+   * @returns {Promise<void>}
+   */
+  submitEvaluationMetrics (span, metrics) {
+    return this.#experiment.submitEvaluationMetrics(span, metrics)
+  }
+
+  /**
+   * @param {object} [options]
+   * @returns {Promise<void>}
+   */
+  close (options) {
+    return this.#experiment.close(options)
+  }
+}
+
+module.exports = { Experiment, ExternalExperiment, normalizeEvaluators, validateEvaluatorName }

package/packages/dd-trace/src/llmobs/experiments/index.js

@@ -1,9 +1,10 @@
 'use strict'
 
 const log = require('../../log')
-const { ExperimentsClient, API_BASE_PATH } = require('./client')
+const { ExperimentsClient } = require('./client')
 const { Dataset, DatasetRecord } = require('./dataset')
-const { Experiment } = require('./experiment')
+const { Experiment, ExternalExperiment } = require('./experiment')
+const { validateTagsList } = require('./util')
 const NoopExperiments = require('./noop')
 
 // Poll `attempt` with exponential backoff until it returns true or the time
@@ -26,17 +27,27 @@ async function retryWithBackoff (attempt, { maxTotalMs = 30_000, baseDelayMs = 2
 // experiments against the LLM Obs backend using the tracer's own config.
 class Experiments {
   #client
+  #config
   #llmobs
   #projectName
 
   constructor (config, llmobs) {
+    this.#config = config
     this.#llmobs = llmobs
     this.#projectName = config.llmobs?.mlApp || config.service
-    this.#client = new ExperimentsClient({
-      apiKey: config.DD_API_KEY,
-      appKey: config.DD_APP_KEY,
-      site: config.site,
-      projectName: this.#projectName,
+    this.#client = this.#clientForProject(this.#projectName)
+  }
+
+  /**
+   * @param {string} projectName
+   * @returns {ExperimentsClient}
+   */
+  #clientForProject (projectName) {
+    return new ExperimentsClient({
+      apiKey: this.#config.DD_API_KEY,
+      appKey: this.#config.DD_APP_KEY,
+      site: this.#config.site,
+      projectName,
     })
   }
 
@@ -56,7 +67,9 @@ class Experiments {
           if (recordIds.has(record.id)) throw new Error(`Duplicate record id '${record.id}'`)
           recordIds.add(record.id)
         }
-        dataset.addRecord(new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id))
+        dataset.addRecord(
+          new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id, record.tags)
+        )
       }
     }
     return dataset
@@ -64,73 +77,48 @@ class Experiments {
 
   // Pull an existing dataset by name (with its records). Polls with exponential
   // backoff to absorb read-after-write lag; pass `expectedRecordCount` to also
-  // wait until that many records are readable.
+  // wait until that many records are readable. Pass `tags` to filter records by
+  // dataset record tags.
   async pullDataset (name, options = {}) {
-    const { expectedRecordCount, maxWaitMs = 30_000, version } = options
+    const { expectedRecordCount, maxWaitMs = 30_000, tags, version } = options
+    const filterTags = validateTagsList(tags)
     const projectId = await this.#client.ensureProjectId()
 
-    let datasetId = null
-    let description = ''
+    let pulledDataset = null
     let records = []
-    let recordIds = []
-    let datasetVersion = version ?? null
+    const datasetVersion = version ?? null
     let latestVersion = null
     let lastError = ''
 
     const succeeded = await retryWithBackoff(async () => {
       try {
-        if (datasetId === null) {
-          const listed = await this.#client.request(
-            'GET',
-            `${API_BASE_PATH}/${projectId}/datasets?filter[name]=${encodeURIComponent(name)}`
-          )
-          const datasets = listed?.data
-          if (datasets) {
-            for (const item of datasets) {
-              if (item?.attributes?.name === name) {
-                datasetId = String(item?.id ?? '')
-                description = String(item?.attributes?.description ?? '')
-                latestVersion = item?.attributes?.current_version ?? null
-                datasetVersion = version ?? latestVersion
-                break
-              }
+        if (pulledDataset === null) {
+          const datasets = await this.#client.listDatasets(projectId, { name })
+          for (const dataset of datasets) {
+            if (dataset.name() === name) {
+              pulledDataset = dataset
+              latestVersion = dataset.latestVersion()
+              break
             }
           }
-          if (datasetId === null) return false
+          if (pulledDataset === null) return false
         }
 
         const recs = []
-        const ids = []
         let cursor = ''
         // Follow the meta.after / page[cursor] pagination until the last page.
         for (;;) {
-          const query = new URLSearchParams()
-          if (cursor) query.set('page[cursor]', cursor)
-          if (datasetVersion !== null) query.set('filter[version]', String(datasetVersion))
           // eslint-disable-next-line no-await-in-loop
-          const resp = await this.#client.request(
-            'GET',
-            `${API_BASE_PATH}/${projectId}/datasets/${datasetId}/records?${query.toString()}`
-          )
-          const recordData = resp?.data
-          if (recordData) {
-            for (const item of recordData) {
-              const attrs = item?.attributes ?? item
-              const recordId = String(item?.id ?? attrs?.id ?? '')
-              recs.push(new DatasetRecord(
-                attrs?.input ?? null,
-                attrs?.expected_output ?? null,
-                attrs?.metadata ?? {},
-                recordId === '' ? null : recordId
-              ))
-              ids.push(recordId)
-            }
-          }
-          cursor = resp?.meta?.after ?? ''
+          const page = await this.#client.listDatasetRecords(projectId, pulledDataset.id(), {
+            cursor,
+            tags: filterTags,
+            version: datasetVersion,
+          })
+          for (const record of page.records) recs.push(record)
+          cursor = page.after
           if (!cursor) break
         }
         records = recs
-        recordIds = ids
         lastError = ''
 
         return expectedRecordCount == null || recs.length >= expectedRecordCount
@@ -140,10 +128,10 @@ class Experiments {
       }
     }, { maxTotalMs: maxWaitMs })
 
-    if (datasetId === null && lastError) {
+    if (pulledDataset === null && lastError) {
       throw new Error(`Failed to list datasets in project '${this.#projectName}': ${lastError}`)
     }
-    if (datasetId === null) {
+    if (pulledDataset === null) {
       throw new Error(`Dataset '${name}' not found in project '${this.#projectName}' (after ${maxWaitMs}ms)`)
     }
     if (!succeeded && lastError) {
@@ -156,16 +144,22 @@ class Experiments {
       )
     }
 
+    for (const record of records) {
+      if (record.id === null || record.id === undefined || record.id === '') {
+        throw new Error(`Failed to pull dataset '${name}': backend returned a record without an id`)
+      }
+    }
+
     return Dataset.fromExisting(
       this.#client,
       name,
-      description,
-      datasetId,
+      pulledDataset.description(),
+      pulledDataset.id(),
       projectId,
       records,
-      recordIds,
       datasetVersion,
-      latestVersion
+      latestVersion,
+      filterTags
     )
   }
 
@@ -173,6 +167,22 @@ class Experiments {
   experiment (options) {
     return new Experiment(this.#client, options, this.#llmobs)
   }
+
+  /**
+   * Start an externally-driven experiment for eval frameworks that already own
+   * task execution. Call submitSpan() once per completed row, then
+   * submitEvaluationMetrics() with the generated span id.
+   *
+   * @param {object} options
+   * @returns {Promise<ExternalExperiment>}
+   */
+  startExperiment (options) {
+    const client = options?.projectName === undefined || options.projectName === this.#projectName
+      ? this.#client
+      : this.#clientForProject(options.projectName)
+    return new Experiment(client, { ...options, external: true }).start()
+      .then(experiment => new ExternalExperiment(experiment))
+  }
 }
 
 // Factory used by the LLMObs SDK: returns a real Experiments instance when
@@ -186,12 +196,13 @@ function createExperiments (config, llmobs) {
     return new NoopExperiments('DD_API_KEY and DD_APP_KEY are required for experiments')
   }
   if (!config.llmobs?.mlApp && !config.service) {
-    log.warn('LLMObs experiments: no project name configured, set DD_LLMOBS_ML_APP or DD_SERVICE')
-    return new NoopExperiments(
-      'no project name configured; set the DD_LLMOBS_ML_APP environment variable (or llmobs.mlApp in ' +
+    const reason = 'no project name configured; set the DD_LLMOBS_ML_APP environment variable (or llmobs.mlApp in ' +
       'tracer.init()) to name the LLM Obs project, or DD_SERVICE (or service in tracer.init()) as a fallback, ' +
       'then retry'
-    )
+    const experiments = new Experiments(config, llmobs)
+    return new NoopExperiments(reason, {
+      startExperiment: (options) => experiments.startExperiment(options),
+    })
   }
   return new Experiments(config, llmobs)
 }