dd-trace 5.122.0 → 5.124.0

package/packages/dd-trace/src/llmobs/experiments/index.js

@@ -2,10 +2,13 @@
 
 const log = require('../../log')
 const { ExperimentsClient } = require('./client')
-const { Dataset, DatasetRecord } = require('./dataset')
+const { Dataset } = require('./dataset')
 const { Experiment, ExternalExperiment } = require('./experiment')
+const { validateTagsList } = require('./util')
 const NoopExperiments = require('./noop')
 
+const DEFAULT_PROJECT_NAME = 'default-project'
+
 // Poll `attempt` with exponential backoff until it returns true or the time
 // budget is spent. Used for eventually-consistent reads (pullDataset).
 async function retryWithBackoff (attempt, { maxTotalMs = 30_000, baseDelayMs = 250, maxDelayMs = 8000 } = {}) {
@@ -32,8 +35,8 @@ class Experiments {
 
   constructor (config, llmobs) {
     this.#config = config
-    this.#llmobs = llmobs
-    this.#projectName = config.llmobs?.mlApp || config.service
+    this.#llmobs = config.llmobs?.mlApp || config.service ? llmobs : undefined
+    this.#projectName = config.llmobs?.projectName || DEFAULT_PROJECT_NAME
     this.#client = this.#clientForProject(this.#projectName)
   }
 
@@ -50,50 +53,54 @@ class Experiments {
     })
   }
 
+  /**
+   * @param {string | undefined} projectName
+   * @returns {ExperimentsClient}
+   */
+  #clientForOperation (projectName) {
+    if (projectName !== undefined && projectName !== this.#projectName) {
+      return this.#clientForProject(projectName)
+    }
+    if (this.#client === undefined) this.#client = this.#clientForProject(projectName)
+    return this.#client
+  }
+
   // Create a local dataset buffer. Pushed remotely on first experiment run.
   createDataset (name, descriptionOrOptions = '') {
     const options = typeof descriptionOrOptions === 'string'
       ? { description: descriptionOrOptions }
       : (descriptionOrOptions ?? {})
-    const dataset = new Dataset(this.#client, name, options.description ?? '')
-    const recordIds = new Set()
-    if ((options.records) != null) {
-      for (const record of options.records) {
-        if (record.id !== undefined && (typeof record.id !== 'string' || record.id.length === 0)) {
-          throw new Error('record id must be a non-empty string')
-        }
-        if (record.id !== undefined) {
-          if (recordIds.has(record.id)) throw new Error(`Duplicate record id '${record.id}'`)
-          recordIds.add(record.id)
-        }
-        dataset.addRecord(new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id))
-      }
-    }
+    const client = this.#clientForOperation(options.projectName)
+    const dataset = new Dataset(client, name, options.description ?? '')
+    if ((options.records) != null) dataset.addRecords(options.records)
     return dataset
   }
 
   // Pull an existing dataset by name (with its records). Polls with exponential
   // backoff to absorb read-after-write lag; pass `expectedRecordCount` to also
-  // wait until that many records are readable.
+  // wait until that many records are readable. Pass `tags` to filter records by
+  // dataset record tags.
   async pullDataset (name, options = {}) {
-    const { expectedRecordCount, maxWaitMs = 30_000, version } = options
-    const projectId = await this.#client.ensureProjectId()
+    const { expectedRecordCount, maxWaitMs = 30_000, projectName, tags, version } = options
+    const filterTags = validateTagsList(tags)
+    const client = this.#clientForOperation(projectName)
+    const resolvedProjectName = projectName ?? this.#projectName
+    const projectId = await client.ensureProjectId()
 
     let pulledDataset = null
     let records = []
-    let datasetVersion = version ?? null
+    const datasetVersion = version ?? null
     let latestVersion = null
     let lastError = ''
 
     const succeeded = await retryWithBackoff(async () => {
       try {
         if (pulledDataset === null) {
-          const datasets = await this.#client.listDatasets(projectId, { name })
+          const datasets = await client.listDatasets(projectId, { name })
           for (const dataset of datasets) {
             if (dataset.name() === name) {
               pulledDataset = dataset
               latestVersion = dataset.latestVersion()
-              datasetVersion = version ?? latestVersion
               break
             }
           }
@@ -105,8 +112,9 @@ class Experiments {
         // Follow the meta.after / page[cursor] pagination until the last page.
         for (;;) {
           // eslint-disable-next-line no-await-in-loop
-          const page = await this.#client.listDatasetRecords(projectId, pulledDataset.id(), {
+          const page = await client.listDatasetRecords(projectId, pulledDataset.id(), {
             cursor,
+            tags: filterTags,
             version: datasetVersion,
           })
           for (const record of page.records) recs.push(record)
@@ -124,13 +132,13 @@ class Experiments {
     }, { maxTotalMs: maxWaitMs })
 
     if (pulledDataset === null && lastError) {
-      throw new Error(`Failed to list datasets in project '${this.#projectName}': ${lastError}`)
+      throw new Error(`Failed to list datasets in project '${resolvedProjectName}': ${lastError}`)
     }
     if (pulledDataset === null) {
-      throw new Error(`Dataset '${name}' not found in project '${this.#projectName}' (after ${maxWaitMs}ms)`)
+      throw new Error(`Dataset '${name}' not found in project '${resolvedProjectName}' (after ${maxWaitMs}ms)`)
     }
     if (!succeeded && lastError) {
-      throw new Error(`Failed to fetch records for dataset '${name}' in project '${this.#projectName}': ${lastError}`)
+      throw new Error(`Failed to fetch records for dataset '${name}' in project '${resolvedProjectName}': ${lastError}`)
     }
     if (!succeeded && expectedRecordCount != null) {
       throw new Error(
@@ -139,21 +147,45 @@ class Experiments {
       )
     }
 
+    for (const record of records) {
+      if (record.id === null || record.id === undefined || record.id === '') {
+        throw new Error(`Failed to pull dataset '${name}': backend returned a record without an id`)
+      }
+    }
+
     return Dataset.fromExisting(
-      this.#client,
+      client,
       name,
       pulledDataset.description(),
       pulledDataset.id(),
       projectId,
       records,
       datasetVersion,
-      latestVersion
+      latestVersion,
+      filterTags
     )
   }
 
-  // Build an experiment: { name, dataset, task, evaluators, description?, config?, tags? }.
+  // Build an experiment with a dataset, task, evaluators, and optional project/config/tags.
   experiment (options) {
-    return new Experiment(this.#client, options, this.#llmobs)
+    const datasetProjectName = options?.dataset?.projectName?.()
+    if (options?.projectName !== undefined &&
+        datasetProjectName !== undefined &&
+        options.projectName !== datasetProjectName) {
+      throw new Error(
+        `Experiment project '${options.projectName}' does not match dataset project '${datasetProjectName}'`
+      )
+    }
+    const projectName = options?.projectName ?? datasetProjectName
+    const client = this.#clientForOperation(projectName)
+    const usesDatasetOverride = datasetProjectName !== undefined && datasetProjectName !== this.#projectName
+    const resolvedProjectName = projectName ?? this.#config.llmobs?.projectName
+    const experimentOptions = options?.projectName === undefined &&
+      (usesDatasetOverride || this.#config.llmobs?.projectName !== undefined) &&
+      resolvedProjectName !== undefined
+      ? { ...options, projectName: resolvedProjectName }
+      : options
+    return new Experiment(client, experimentOptions, this.#llmobs)
   }
 
   /**
@@ -165,10 +197,11 @@ class Experiments {
    * @returns {Promise<ExternalExperiment>}
    */
   startExperiment (options) {
-    const client = options?.projectName === undefined || options.projectName === this.#projectName
-      ? this.#client
-      : this.#clientForProject(options.projectName)
-    return new Experiment(client, { ...options, external: true }).start()
+    const client = this.#clientForOperation(options?.projectName)
+    const experimentOptions = options?.projectName === undefined && this.#config.llmobs?.projectName !== undefined
+      ? { ...options, projectName: this.#config.llmobs.projectName }
+      : options
+    return new Experiment(client, { ...experimentOptions, external: true }).start()
       .then(experiment => new ExternalExperiment(experiment))
   }
 }
@@ -183,15 +216,6 @@ function createExperiments (config, llmobs) {
     log.warn('LLMObs experiments: missing api and/or app keys, set DD_API_KEY and DD_APP_KEY')
     return new NoopExperiments('DD_API_KEY and DD_APP_KEY are required for experiments')
   }
-  if (!config.llmobs?.mlApp && !config.service) {
-    const reason = 'no project name configured; set the DD_LLMOBS_ML_APP environment variable (or llmobs.mlApp in ' +
-      'tracer.init()) to name the LLM Obs project, or DD_SERVICE (or service in tracer.init()) as a fallback, ' +
-      'then retry'
-    const experiments = new Experiments(config, llmobs)
-    return new NoopExperiments(reason, {
-      startExperiment: (options) => experiments.startExperiment(options),
-    })
-  }
   return new Experiments(config, llmobs)
 }
 

package/packages/dd-trace/src/llmobs/experiments/noop.js

@@ -7,24 +7,60 @@ const NOOP_EXPERIMENT_ID = '00000000-0000-0000-0000-000000000000'
 const NOOP_SPAN_ID = '0000000000000000'
 const NOOP_TRACE_ID = '00000000000000000000000000000000'
 
+/**
+ * @typedef {object} DatasetRecordNew
+ * @property {string} [id]
+ * @property {unknown} inputData
+ * @property {unknown} [expectedOutput]
+ * @property {Record<string, unknown>} [metadata]
+ * @property {string[]} [tags]
+ */
+
 class NoopDataset {
   #name
   #description
   #records
+  #filterTags
 
   constructor (name = '', options = {}) {
     this.#name = name
     this.#description = typeof options === 'string' ? options : (options.description ?? '')
+    this.#filterTags = typeof options === 'string' ? [] : [...(options.filterTags ?? [])]
     this.#records = (typeof options === 'string' ? [] : (options.records ?? [])).map(record => ({
       id: record.id ?? null,
       input: record.inputData,
       expectedOutput: record.expectedOutput ?? null,
       metadata: record.metadata ?? {},
+      tags: [...(record.tags ?? [])],
     }))
   }
 
-  addRecord (input, expectedOutput, metadata) {
-    this.#records.push({ id: null, input, expectedOutput: expectedOutput ?? null, metadata: metadata ?? {} })
+  addRecord (input, expectedOutput, metadata, tags) {
+    this.#records.push({
+      id: null,
+      input,
+      expectedOutput: expectedOutput ?? null,
+      metadata: metadata ?? {},
+      tags: [...(tags ?? [])],
+    })
+    return this
+  }
+
+  /**
+   * Add multiple records to a dataset.
+   * @param {DatasetRecordNew[]} records
+   * @returns {NoopDataset} This dataset for chaining.
+   */
+  addRecords (records) {
+    for (const record of records) {
+      this.#records.push({
+        id: record.id ?? null,
+        input: record.inputData,
+        expectedOutput: record.expectedOutput ?? null,
+        metadata: record.metadata ?? {},
+        tags: [...(record.tags ?? [])],
+      })
+    }
     return this
   }
 
@@ -46,6 +82,49 @@ class NoopDataset {
     return Promise.resolve({ pushedCount: 0, totalCount: 0 })
   }
 
+  /**
+   * Add tags to a dataset record.
+   * @param {number} index Dataset record index.
+   * @param {string[]} tags Tags in key:value format.
+   * @returns {NoopDataset} This dataset for chaining.
+   */
+  addTags (index, tags) {
+    const record = this.#records[index]
+    if (!record) return this
+    const normalizedTags = tags ?? []
+    record.tags = [...new Set([...(record.tags ?? []), ...normalizedTags])].sort()
+    return this
+  }
+
+  /**
+   * Remove tags from a dataset record.
+   * @param {number} index Dataset record index.
+   * @param {string[]} tags Tags in key:value format.
+   * @returns {NoopDataset} This dataset for chaining.
+   */
+  removeTags (index, tags) {
+    const record = this.#records[index]
+    if (!record) return this
+    const normalizedTags = tags ?? []
+    const removed = new Set(normalizedTags)
+    record.tags = (record.tags ?? []).filter(tag => !removed.has(tag)).sort()
+    return this
+  }
+
+  /**
+   * Replace all tags on a dataset record.
+   * @param {number} index Dataset record index.
+   * @param {string[]} tags Tags in key:value format.
+   * @returns {NoopDataset} This dataset for chaining.
+   */
+  replaceTags (index, tags) {
+    const record = this.#records[index]
+    if (!record) return this
+    const normalizedTags = tags ?? []
+    record.tags = [...normalizedTags]
+    return this
+  }
+
   name () {
     return this.#name
   }
@@ -62,6 +141,10 @@ class NoopDataset {
     return null
   }
 
+  projectName () {
+    return null
+  }
+
   version () {
     return null
   }
@@ -70,6 +153,14 @@ class NoopDataset {
     return null
   }
 
+  /**
+   * Return the tags used to filter this dataset.
+   * @returns {string[]} Dataset record filter tags.
+   */
+  filterTags () {
+    return [...this.#filterTags]
+  }
+
   records () {
     return [...this.#records]
   }
@@ -105,7 +196,13 @@ class NoopExperiment {
   }
 
   run () {
-    return Promise.resolve({ experimentId: null, rows: [], url: null })
+    return Promise.resolve({
+      experimentId: null,
+      rows: [],
+      summaryEvaluations: {},
+      runs: [],
+      url: null,
+    })
   }
 
   /**
@@ -140,11 +237,9 @@ class NoopExperiment {
 // throwing, so intentionally disabled experiments remain graceful.
 class NoopExperiments {
   #reason
-  #startExperiment
 
-  constructor (reason, options = {}) {
+  constructor (reason) {
     this.#reason = reason || 'LLMObs experiments are not available'
-    this.#startExperiment = options.startExperiment
   }
 
   #warn () {
@@ -156,9 +251,9 @@ class NoopExperiments {
     return new NoopDataset(name, options)
   }
 
-  pullDataset (name) {
+  pullDataset (name, options = {}) {
     this.#warn()
-    return Promise.resolve(new NoopDataset(name))
+    return Promise.resolve(new NoopDataset(name, { filterTags: options.tags }))
   }
 
   experiment (options = {}) {
@@ -171,10 +266,6 @@ class NoopExperiments {
    * @returns {Promise<ExternalExperiment>}
    */
   startExperiment (options = {}) {
-    if (this.#startExperiment !== undefined && options.projectName) {
-      return this.#startExperiment(options)
-    }
-
     this.#warn()
     return Promise.resolve(new ExternalExperiment(new NoopExperiment(options.name, true)))
   }

package/packages/dd-trace/src/llmobs/experiments/result.js

@@ -27,6 +27,7 @@ class ExperimentRun {
   constructor (fields) {
     this.runId = fields.runId
     this.runIteration = fields.runIteration
+    this.hasError = fields.hasError
     this.rows = fields.rows
     this.summaryEvaluations = fields.summaryEvaluations
   }

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -1,5 +1,7 @@
 'use strict'
 
+const { randomUUID } = require('node:crypto')
+
 const log = require('../../log')
 
 // Matches the backend and dd-trace-py evaluator metric label contract.
@@ -22,6 +24,43 @@ function hasEntries (value) {
   return false
 }
 
+function validateTagsList (tags) {
+  if (tags == null) return []
+  if (!Array.isArray(tags)) throw new TypeError('Tags must be an array of strings')
+  for (const tag of tags) {
+    if (typeof tag !== 'string') throw new TypeError('Each tag must be a string')
+    if (tag.indexOf(':') <= 0) {
+      throw new Error(`Tag '${tag}' is malformed. Tags must be in 'key:value' format (e.g., 'env:prod').`)
+    }
+  }
+  return [...tags]
+}
+
+function tagOperationsAreEmpty (operations) {
+  return operations == null || (
+    !Object.hasOwn(operations, 'replace') &&
+    !Object.hasOwn(operations, 'add') &&
+    !Object.hasOwn(operations, 'remove')
+  )
+}
+
+/**
+ * @param {unknown} value
+ * @param {string} name
+ * @returns {number}
+ */
+function normalizePositiveInteger (value, name) {
+  if (!Number.isInteger(value) || value < 1) throw new Error(`${name} must be a positive integer`)
+  return value
+}
+
+/**
+ * @returns {string}
+ */
+function generateRunId () {
+  return randomUUID()
+}
+
 /**
  * @param {string} name
  */
@@ -165,16 +204,20 @@ function stringify (value) {
  * @returns {string[]}
  */
 function buildTags (userTags, autoTags) {
-  const tags = new Map()
+  const tagsByKey = new Map()
   if ((userTags) != null) {
     for (const [key, value] of Object.entries(userTags)) {
-      tags.set(key, `${key}:${value}`)
+      const values = Array.isArray(value) ? value : [value]
+      tagsByKey.set(key, values.map(item => `${key}:${item}`))
     }
   }
   for (const [key, value] of Object.entries(autoTags)) {
-    if (value !== undefined && value !== null && value !== '') tags.set(key, `${key}:${value}`)
+    if (value !== undefined && value !== null && value !== '') tagsByKey.set(key, [`${key}:${value}`])
   }
-  return [...tags.values()]
+
+  const tags = []
+  for (const values of tagsByKey.values()) tags.push(...values)
+  return tags
 }
 
 /**
@@ -186,6 +229,35 @@ function mergeTags (baseTags, overrideTags) {
   return { ...baseTags, ...overrideTags }
 }
 
+/**
+ * @param {string[] | undefined} tags
+ * @returns {Record<string, string | string[]>}
+ */
+function recordTagsToObject (tags) {
+  const result = {}
+  if (!Array.isArray(tags)) return result
+  for (const tag of tags) {
+    const separator = tag.indexOf(':')
+    if (separator <= 0) continue
+
+    const key = tag.slice(0, separator)
+    const value = tag.slice(separator + 1)
+    if (!Object.hasOwn(result, key)) {
+      Object.defineProperty(result, key, {
+        configurable: true,
+        enumerable: true,
+        value,
+        writable: true,
+      })
+    } else if (Array.isArray(result[key])) {
+      result[key].push(value)
+    } else {
+      result[key] = [result[key], value]
+    }
+  }
+  return result
+}
+
 /**
  * @param {number} ms
  * @returns {Promise<void>}
@@ -244,13 +316,18 @@ module.exports = {
   buildSpanMetadata,
   buildTags,
   durationNs,
+  generateRunId,
   hasEntries,
   inferMetricType,
   mergeTags,
   normalizeEvaluators,
   normalizeJsonMetricValue,
+  normalizePositiveInteger,
+  recordTagsToObject,
   sleep,
   stringify,
+  tagOperationsAreEmpty,
   timestampMs,
   validateEvaluatorName,
+  validateTagsList,
 }

package/packages/dd-trace/src/llmobs/index.js

@@ -3,7 +3,9 @@
 const { channel } = require('dc-polyfill')
 
 const { readDatadogTags, writeDatadogTags } = require('../carrier')
+const { registerTelemetryFlusher } = require('../flush')
 const log = require('../log')
+const { createServerlessDeliveryTracker } = require('../serverless')
 const { DD_MAJOR } = require('../../../../version')
 const startupLogs = require('../startup-log')
 const {
@@ -57,6 +59,8 @@ let spanWriter
 /** @type {LLMObsEvalMetricsWriter | null} */
 let evalWriter
 
+let unregisterTelemetryFlusher
+
 /** @type {import('../config/config-base')} */
 let globalTracerConfig
 
@@ -66,23 +70,36 @@ let globalTracerConfig
 function enable (config) {
   globalTracerConfig = config
 
+  const retiredSpanWriter = spanWriter
+  const retiredEvalWriter = evalWriter
+  const isReinitializing = Boolean(retiredSpanWriter || retiredEvalWriter)
+  unregisterTelemetryFlusher?.()
+  retireWriters(retiredSpanWriter, retiredEvalWriter)
+
   const startTime = performance.now()
   // create writers and eval writer append and flush channels
   // span writer append is handled by the span processor
   evalWriter = new LLMObsEvalMetricsWriter(config)
   spanWriter = new LLMObsSpanWriter(config)
+  const currentEvalWriter = evalWriter
+  const currentSpanWriter = spanWriter
+  unregisterTelemetryFlusher = registerTelemetryFlusher(done => {
+    flushWriters(done, currentSpanWriter, currentEvalWriter)
+  })
 
-  evalMetricAppendCh.subscribe(handleEvalMetricAppend)
-  flushCh.subscribe(handleFlush)
-  registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
+  if (!isReinitializing) {
+    evalMetricAppendCh.subscribe(handleEvalMetricAppend)
+    flushCh.subscribe(handleFlush)
+    registerUserSpanProcessorCh.subscribe(handleRegisterProcessor)
+  }
 
   // span processing
   spanProcessor = new LLMObsSpanProcessor(config)
   spanProcessor.setWriter(spanWriter)
-  spanFinishCh.subscribe(handleSpanProcess)
+  if (!isReinitializing) spanFinishCh.subscribe(handleSpanProcess)
 
   // distributed tracing for llmobs
-  injectCh.subscribe(handleLLMObsInjection)
+  if (!isReinitializing) injectCh.subscribe(handleLLMObsInjection)
 
   setAgentStrategy(config, useAgentless => {
     if (useAgentless && !(config.DD_API_KEY && config.site)) {
@@ -94,8 +111,10 @@ function enable (config) {
       }
     }
 
-    evalWriter?.setAgentless(useAgentless)
-    spanWriter?.setAgentless(useAgentless)
+    // A disable can happen while transport selection is still pending. Keep
+    // configuring these writers so their queued lifecycle flushes can drain.
+    currentEvalWriter.setAgentless(useAgentless)
+    currentSpanWriter.setAgentless(useAgentless)
 
     telemetry.recordLLMObsEnabled(startTime, config)
     log.debug('[LLMObs] Enabled LLM Observability with configuration: %o', config.llmobs)
@@ -109,16 +128,39 @@ function disable () {
   if (injectCh.hasSubscribers) injectCh.unsubscribe(handleLLMObsInjection)
   if (registerUserSpanProcessorCh.hasSubscribers) registerUserSpanProcessorCh.unsubscribe(handleRegisterProcessor)
 
-  spanWriter?.destroy()
-  evalWriter?.destroy()
+  const retiredSpanWriter = spanWriter
+  const retiredEvalWriter = evalWriter
   spanProcessor?.setWriter(null)
+  unregisterTelemetryFlusher?.()
+  unregisterTelemetryFlusher = undefined
 
   spanWriter = null
   evalWriter = null
 
+  retireWriters(retiredSpanWriter, retiredEvalWriter)
+
   log.debug('[LLMObs] Disabled LLM Observability')
 }
 
+/**
+ * Keeps retired writers reachable until their destroy-triggered deliveries complete.
+ * @param {LLMObsSpanWriter | null} retiredSpanWriter
+ * @param {LLMObsEvalMetricsWriter | null} retiredEvalWriter
+ * @returns {void}
+ */
+function retireWriters (retiredSpanWriter, retiredEvalWriter) {
+  const retiredWriters = [retiredSpanWriter, retiredEvalWriter].filter(Boolean)
+  if (retiredWriters.length === 0) return
+  let remainingWriters = retiredWriters.length
+  const unregisterRetiredFlusher = registerTelemetryFlusher(done => {
+    flushWriters(done, retiredSpanWriter, retiredEvalWriter)
+  })
+  function onWriterDestroyed () {
+    if (--remainingWriters === 0) unregisterRetiredFlusher?.()
+  }
+  for (const writer of retiredWriters) writer.destroy(onWriterDestroyed)
+}
+
 // since LLMObs traces can extend between services and be the same trace,
 // we need to propagate the parent id, mlApp, session id, and sampling rate/decision.
 function handleLLMObsInjection ({ carrier }) {
@@ -192,15 +234,36 @@ function handleLLMObsInjection ({ carrier }) {
   if (tags !== existing) writeDatadogTags(carrier, tags)
 }
 
-function handleFlush () {
-  let err = ''
-  try {
-    spanWriter.flush()
-    evalWriter.flush()
-  } catch (e) {
-    err = 'writer_flush_error'
-    log.warn('Failed to flush LLMObs spans and evaluation metrics:', e.message)
+/**
+ * Flushes the specified LLMObs writers and joins deliveries active at the boundary.
+ * @param {Function} [done]
+ * @param {LLMObsSpanWriter | null} [currentSpanWriter]
+ * @param {LLMObsEvalMetricsWriter | null} [currentEvalWriter]
+ * @returns {boolean} `true` when a writer throws synchronously.
+ */
+function flushWriters (done, currentSpanWriter = spanWriter, currentEvalWriter = evalWriter) {
+  let failed = false
+  const deliveryTracker = createServerlessDeliveryTracker()
+  const flush = writer => {
+    try {
+      if (deliveryTracker && writer) deliveryTracker.track(complete => writer.flush(complete))
+      // Non-serverless flushes retain the existing writer behavior.
+      else writer?.flush()
+    } catch (error) {
+      failed = true
+      log.warn('Failed to flush LLMObs writer:', error.message)
+    }
   }
+
+  flush(currentSpanWriter)
+  flush(currentEvalWriter)
+  deliveryTracker?.waitForIdle(done)
+  if (!deliveryTracker) done?.()
+  return failed
+}
+
+function handleFlush () {
+  const err = flushWriters() ? 'writer_flush_error' : ''
   telemetry.recordUserFlush(err)
 }