dd-trace 6.19.0 → 6.20.0

package/packages/dd-trace/src/llmobs/experiments/remote-evaluator.js

@@ -0,0 +1,110 @@
+'use strict'
+
+const { BaseEvaluator, EvaluatorResult } = require('./evaluator')
+const { hasEntries } = require('./util')
+
+/**
+ * @typedef {{value?: unknown, reasoning?: string, assessment?: string, status?: string}} RemoteEvaluatorResponse
+ * @typedef {{evaluatorInfer: (evalName: string, context: object) => Promise<RemoteEvaluatorResponse>}}
+ *   RemoteEvaluatorClient
+ */
+
+/**
+ * @param {import('./evaluator').EvaluatorContext} context
+ */
+function defaultContextTransform (context) {
+  const transformed = {
+    span_input: context.inputData,
+    span_output: context.outputData,
+  }
+
+  const meta = {}
+  let hasMeta = false
+  if (context.expectedOutput != null) {
+    meta.expected_output = context.expectedOutput
+    hasMeta = true
+  }
+  if (hasEntries(context.metadata)) {
+    meta.metadata = context.metadata
+    hasMeta = true
+  }
+  if (hasMeta) transformed.meta = meta
+
+  if (context.spanId) transformed.span_id = context.spanId
+  if (context.traceId) transformed.trace_id = context.traceId
+
+  return transformed
+}
+
+class RemoteEvaluatorResult extends EvaluatorResult {
+  /**
+   * @param {unknown} value
+   * @param {{reasoning?: string, assessment?: string, status?: string}} [options]
+   */
+  constructor (value, { reasoning, assessment, status } = {}) {
+    super(value, { reasoning, assessment })
+    this.status = status
+    this.evalSourceType = 'managed'
+  }
+}
+
+class RemoteEvaluatorError extends Error {
+  /**
+   * @param {string} message
+   * @param {{status?: string, backendError?: object}} [options]
+   */
+  constructor (message, { status = 'ERROR', backendError = {} } = {}) {
+    super(message)
+    this.name = 'RemoteEvaluatorError'
+    this.status = status
+    this.backendError = backendError
+  }
+}
+
+/**
+ * Evaluator that references an LLM-as-a-judge evaluator configured in Datadog.
+ */
+class RemoteEvaluator extends BaseEvaluator {
+  #evalName
+  #transformFn
+
+  /**
+   * @param {{evalName: string, transformFn?: (context: import('./evaluator').EvaluatorContext) => object}} options
+   */
+  constructor (options) {
+    super()
+    const { evalName, transformFn } = options ?? {}
+    if (typeof evalName !== 'string' || evalName.trim() === '') {
+      throw new TypeError('evalName must be a non-empty string')
+    }
+    if (transformFn !== undefined && typeof transformFn !== 'function') {
+      throw new TypeError('transformFn must be a function')
+    }
+
+    this.name = evalName.trim()
+    this.#evalName = this.name
+    this.#transformFn = transformFn ?? defaultContextTransform
+  }
+
+  /**
+   * @param {import('./evaluator').EvaluatorContext} context
+   * @param {RemoteEvaluatorClient} [client]
+   */
+  evaluate (context, client) {
+    if (client === undefined || typeof client.evaluatorInfer !== 'function') {
+      throw new Error('RemoteEvaluator can only be evaluated as part of an experiment')
+    }
+
+    const transformed = this.#transformFn(context)
+    return client.evaluatorInfer(this.#evalName, transformed).then(result => new RemoteEvaluatorResult(
+      result?.value,
+      {
+        reasoning: result?.reasoning,
+        assessment: result?.assessment,
+        status: result?.status,
+      }
+    ))
+  }
+}
+
+module.exports = { RemoteEvaluator, RemoteEvaluatorError }

package/packages/dd-trace/src/llmobs/experiments/util.js

@@ -62,6 +62,7 @@ function generateRunId () {
 function validateEvaluatorName (name) {
   if (typeof name !== 'string') throw new TypeError('Evaluator name must be a string')
   if (name.length === 0) throw new Error('Evaluator name cannot be empty')
+  if (name === '__proto__') throw new Error("Evaluator name '__proto__' is reserved")
   if (!EVALUATOR_NAME_PATTERN.test(name)) {
     throw new Error(
       `Evaluator name '${name}' is invalid. Name must contain only alphanumeric characters, underscores, and hyphens.`
@@ -77,10 +78,34 @@ function functionName (fn, fallback) {
   return typeof fn.name === 'string' && fn.name.length > 0 ? fn.name : fallback
 }
 
+/**
+ * @param {unknown} evaluator
+ * @param {string} kind
+ */
+function isClassEvaluator (evaluator, kind) {
+  // Lazy loading avoids a cycle: evaluator.js uses validateEvaluatorName from this module.
+  const { BaseEvaluator, BaseSummaryEvaluator } = require('./evaluator')
+  return kind === 'summary'
+    ? evaluator instanceof BaseSummaryEvaluator
+    : evaluator instanceof BaseEvaluator
+}
+
+/**
+ * @param {unknown} evaluator
+ * @param {string} kind
+ * @param {number} index
+ */
+function evaluatorName (evaluator, kind, index) {
+  if (isClassEvaluator(evaluator, kind)) return evaluator.name
+  if (typeof evaluator === 'function') return functionName(evaluator, `${kind}_evaluator_${index}`)
+  const baseName = kind === 'summary' ? 'BaseSummaryEvaluator' : 'BaseEvaluator'
+  throw new TypeError(`${kind} evaluator must be a function or a ${baseName} instance`)
+}
+
 /**
  * @param {unknown} evaluators
  * @param {string} kind
- * @returns {Array<[string, (...args: unknown[]) => unknown]>}
+ * @returns {Array<[string, Function | object]>}
  */
 function normalizeEvaluators (evaluators, kind) {
   if (evaluators == null) return []
@@ -90,8 +115,7 @@ function normalizeEvaluators (evaluators, kind) {
     const indexesByName = new Map()
     for (let i = 0; i < evaluators.length; i++) {
       const evaluator = evaluators[i]
-      if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator must be a function`)
-      const name = functionName(evaluator, `${kind}_evaluator_${i}`)
+      const name = evaluatorName(evaluator, kind, i)
       validateEvaluatorName(name)
       if (indexesByName.has(name)) {
         log.warn('Duplicate %s evaluator name %s; previous evaluator will be overwritten', kind, name)
@@ -104,13 +128,17 @@ function normalizeEvaluators (evaluators, kind) {
     return normalized
   }
 
-  if (typeof evaluators !== 'object') {
-    throw new TypeError(`${kind} evaluators must be an array of functions or an object keyed by evaluator name`)
+  if (typeof evaluators !== 'object' || evaluators === null) {
+    throw new TypeError(
+      `${kind} evaluators must be an array of functions or class instances, or an object keyed by evaluator name`
+    )
   }
 
   for (const [name, evaluator] of Object.entries(evaluators)) {
     validateEvaluatorName(name)
-    if (typeof evaluator !== 'function') throw new TypeError(`${kind} evaluator '${name}' must be a function`)
+    if (!isClassEvaluator(evaluator, kind) && typeof evaluator !== 'function') {
+      throw new TypeError(`${kind} evaluator '${name}' must be a function or a class instance`)
+    }
     normalized.push([name, evaluator])
   }
   return normalized

package/packages/dd-trace/src/llmobs/plugins/openai/index.js

@@ -94,8 +94,8 @@ class OpenAiLLMObsPlugin extends LLMObsPlugin {
     if (!methodName) return // we will not trace all openai methods for llmobs
 
     const inputs = ctx.args[0] // completion, chat completion, and embeddings take one argument
-    const response = ctx.result?.data // no result if error
-    const error = !!span.context().getTag('error')
+    const response = ctx.result?.data // no result if error, or if a stream ended before any response arrived
+    const error = !!span.context().getTag('error') || response == null
 
     const operation = getOperation(methodName)
 

package/packages/dd-trace/src/llmobs/sdk.js

@@ -7,6 +7,8 @@ const logger = require('../log')
 const { getValueFromEnvSources } = require('../config/helper')
 const Span = require('../opentracing/span')
 const {
+  EXPERIMENT_INPUT,
+  EXPERIMENT_OUTPUT,
   SPAN_KIND,
   OUTPUT_VALUE,
   INPUT_VALUE,
@@ -300,13 +302,18 @@ class LLMObs extends NoopLLMObs {
 
       const { inputData, outputData, metadata, metrics, tags, prompt, costTags, toolDefinitions } = options
 
-      if (inputData || outputData) {
+      const hasInputOrOutput = spanKind === 'experiment'
+        ? inputData !== undefined || outputData !== undefined
+        : inputData || outputData
+      if (hasInputOrOutput) {
         if (spanKind === 'llm') {
           this._tagger.tagLLMIO(span, inputData, outputData)
         } else if (spanKind === 'embedding') {
           this._tagger.tagEmbeddingIO(span, inputData, outputData)
         } else if (spanKind === 'retrieval') {
           this._tagger.tagRetrievalIO(span, inputData, outputData)
+        } else if (spanKind === 'experiment') {
+          this._tagger.tagExperimentIO(span, inputData, outputData)
         } else {
           this._tagger.tagTextIO(span, inputData, outputData)
         }
@@ -638,11 +645,20 @@ class LLMObs extends NoopLLMObs {
 
   #autoAnnotate (span, kind, input, output) {
     const annotations = {}
-    if (input && !['llm', 'embedding'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[INPUT_VALUE]) {
+    const spanTags = LLMObsTagger.tagMap.get(span)
+    const isExperiment = kind === 'experiment'
+    const inputKey = isExperiment ? EXPERIMENT_INPUT : INPUT_VALUE
+    const outputKey = isExperiment ? EXPERIMENT_OUTPUT : OUTPUT_VALUE
+    const hasInput = isExperiment ? input !== undefined : input
+    const hasOutput = isExperiment ? output !== undefined : output
+    const hasInputTag = spanTags !== undefined && Object.hasOwn(spanTags, inputKey)
+    const hasOutputTag = spanTags !== undefined && Object.hasOwn(spanTags, outputKey)
+
+    if (hasInput && !['llm', 'embedding'].includes(kind) && !hasInputTag) {
       annotations.inputData = input
     }
 
-    if (output && !['llm', 'retrieval'].includes(kind) && !LLMObsTagger.tagMap.get(span)?.[OUTPUT_VALUE]) {
+    if (hasOutput && !['llm', 'retrieval'].includes(kind) && !hasOutputTag) {
       annotations.outputData = output
     }
 

package/packages/dd-trace/src/llmobs/span_processor.js

@@ -16,6 +16,8 @@ const {
   METADATA,
   COST_TAGS,
   TOOL_DEFINITIONS,
+  EXPERIMENT_INPUT,
+  EXPERIMENT_OUTPUT,
   INPUT_MESSAGES,
   INPUT_VALUE,
   INTEGRATION,
@@ -182,8 +184,13 @@ class LLMObsSpanProcessor {
     }
 
     const llmObsSpan = new LLMObservabilitySpan(spanKind)
+    const isExperiment = spanKind === 'experiment'
+    const hasExperimentInput = isExperiment && Object.hasOwn(mlObsTags, EXPERIMENT_INPUT)
+    const hasExperimentOutput = isExperiment && Object.hasOwn(mlObsTags, EXPERIMENT_OUTPUT)
 
-    if (spanKind === 'llm' && mlObsTags[INPUT_MESSAGES]) {
+    if (hasExperimentInput) {
+      llmObsSpan.input = [{ role: '', content: mlObsTags[EXPERIMENT_INPUT] }]
+    } else if (spanKind === 'llm' && mlObsTags[INPUT_MESSAGES]) {
       llmObsSpan.input = mlObsTags[INPUT_MESSAGES]
       inputType = 'messages'
     } else if (spanKind === 'embedding' && mlObsTags[INPUT_DOCUMENTS]) {
@@ -194,7 +201,9 @@ class LLMObsSpanProcessor {
       inputType = 'value'
     }
 
-    if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) {
+    if (hasExperimentOutput) {
+      llmObsSpan.output = [{ role: '', content: mlObsTags[EXPERIMENT_OUTPUT] }]
+    } else if (spanKind === 'llm' && mlObsTags[OUTPUT_MESSAGES]) {
       llmObsSpan.output = mlObsTags[OUTPUT_MESSAGES]
       outputType = 'messages'
     } else if (spanKind === 'retrieval' && mlObsTags[OUTPUT_DOCUMENTS]) {
@@ -226,34 +235,41 @@ class LLMObsSpanProcessor {
     const processedSpan = this.#runProcessor(llmObsSpan)
     if (processedSpan === undefined) return null
 
-    if (processedSpan.input) {
-      if (inputType === 'messages') {
-        input.messages = processedSpan.input
-      } else if (inputType === 'value') {
-        input.value = processedSpan.input[0].content
-      } else if (inputType === 'documents') {
-        input.documents = processedSpan.input.map((processedDocument, processedDocumentIdx) => ({
-          ...mlObsTags[INPUT_DOCUMENTS][processedDocumentIdx],
-          text: processedDocument.content,
-        }))
+    if (isExperiment) {
+      const [processedInput] = processedSpan.input
+      const [processedOutput] = processedSpan.output
+      if (hasExperimentInput && processedInput !== undefined) meta.input = processedInput.content
+      if (hasExperimentOutput && processedOutput !== undefined) meta.output = processedOutput.content
+    } else {
+      if (processedSpan.input) {
+        if (inputType === 'messages') {
+          input.messages = processedSpan.input
+        } else if (inputType === 'value') {
+          input.value = processedSpan.input[0].content
+        } else if (inputType === 'documents') {
+          input.documents = processedSpan.input.map((processedDocument, processedDocumentIdx) => ({
+            ...mlObsTags[INPUT_DOCUMENTS][processedDocumentIdx],
+            text: processedDocument.content,
+          }))
+        }
       }
-    }
 
-    if (processedSpan.output) {
-      if (outputType === 'messages') {
-        output.messages = processedSpan.output
-      } else if (outputType === 'value') {
-        output.value = processedSpan.output[0].content
-      } else if (outputType === 'documents') {
-        output.documents = processedSpan.output.map((processedDocument, processedDocumentIdx) => ({
-          ...mlObsTags[OUTPUT_DOCUMENTS][processedDocumentIdx],
-          text: processedDocument.content,
-        }))
+      if (processedSpan.output) {
+        if (outputType === 'messages') {
+          output.messages = processedSpan.output
+        } else if (outputType === 'value') {
+          output.value = processedSpan.output[0].content
+        } else if (outputType === 'documents') {
+          output.documents = processedSpan.output.map((processedDocument, processedDocumentIdx) => ({
+            ...mlObsTags[OUTPUT_DOCUMENTS][processedDocumentIdx],
+            text: processedDocument.content,
+          }))
+        }
       }
-    }
 
-    if (input) meta.input = input
-    if (output) meta.output = output
+      meta.input = input
+      meta.output = output
+    }
 
     const prompt = mlObsTags[INPUT_PROMPT]
     if (prompt && spanKind === 'llm') {

package/packages/dd-trace/src/llmobs/tagger.js

@@ -10,6 +10,8 @@ const {
   SESSION_ID_TRACE_DEFAULT_KEY,
   ML_APP,
   SPAN_KIND,
+  EXPERIMENT_INPUT,
+  EXPERIMENT_OUTPUT,
   INPUT_VALUE,
   OUTPUT_DOCUMENTS,
   INPUT_DOCUMENTS,
@@ -273,6 +275,18 @@ class LLMObsTagger {
     this.#tagDocuments(span, outputData, OUTPUT_DOCUMENTS)
   }
 
+  /**
+   * Tags arbitrary JSON-compatible experiment input and output without converting structured values to text.
+   *
+   * @param {import('../opentracing/span')} span
+   * @param {unknown} inputData
+   * @param {unknown} outputData
+   */
+  tagExperimentIO (span, inputData, outputData) {
+    this.#tagExperimentValue(span, inputData, EXPERIMENT_INPUT, 'input')
+    this.#tagExperimentValue(span, outputData, EXPERIMENT_OUTPUT, 'output')
+  }
+
   tagTextIO (span, inputData, outputData) {
     this.#tagText(span, inputData, INPUT_VALUE)
     this.#tagText(span, outputData, OUTPUT_VALUE)
@@ -562,6 +576,27 @@ class LLMObsTagger {
     }
   }
 
+  /**
+   * Validates and stores one free-form experiment I/O value.
+   *
+   * @param {import('../opentracing/span')} span
+   * @param {unknown} data
+   * @param {string} key
+   * @param {string} type
+   */
+  #tagExperimentValue (span, data, key, type) {
+    if (data === undefined) return
+
+    try {
+      if (JSON.stringify(data) !== undefined) {
+        this._setTag(span, key, data)
+        return
+      }
+    } catch {}
+
+    this.#handleFailure(`Failed to parse ${type} value, must be JSON serializable.`, 'invalid_io_text')
+  }
+
   #tagDocuments (span, data, key) {
     if (!data) {
       return

package/packages/dd-trace/src/llmobs/writers/base.js

@@ -293,12 +293,7 @@ class BaseLLMObsWriter {
   }
 
   _encode (payload) {
-    return JSON.stringify(payload, (key, value) => {
-      if (typeof value === 'string') {
-        return encodeUnicode(value) // serialize unicode characters
-      }
-      return value
-    }).replaceAll(String.raw`\\u`, String.raw`\u`) // remove double escaping
+    return encodeUnicode(JSON.stringify(payload))
   }
 }
 

package/packages/dd-trace/src/openfeature/constants/constants.js

@@ -19,6 +19,16 @@ module.exports = {
    */
   EVP_EVENT_SIZE_LIMIT: (1 << 20) - 1024,
 
+  FLAG_EVALUATION_ENDPOINT: '/api/v2/flagevaluation',
+  FLAG_EVALUATION_FLUSH_INTERVAL: 10_000,
+  FLAG_EVALUATION_QUEUE_CAP: 4096,
+  FLAG_EVALUATION_GLOBAL_CAP: 131_072,
+  FLAG_EVALUATION_PER_FLAG_CAP: 10_000,
+  FLAG_EVALUATION_DEGRADED_CAP: 32_768,
+
+  // ECMAScript Date's maximum absolute time value, in milliseconds.
+  MAX_EVALUATION_TIMESTAMP_MS: 8_640_000_000_000_000,
+
   /**
    * @constant
    * @type {string} Channel name for exposure event submission

package/packages/dd-trace/src/openfeature/flagging_provider.js

@@ -8,6 +8,7 @@ const configurationSource = require('./configuration_source')
 const { EXPOSURE_CHANNEL } = require('./constants/constants')
 const EvalMetricsHook = require('./eval-metrics-hook')
 const SpanEnrichmentHook = require('./span-enrichment-hook')
+const FlagEvalEVPHook = require('./writers/flag-eval-evp-hook')
 
 /**
  * OpenFeature provider that integrates with Datadog's feature flagging system.
@@ -17,6 +18,9 @@ class FlaggingProvider extends DatadogNodeServerProvider {
   /** @type {SpanEnrichmentHook | undefined} */
   #spanEnrichmentHook
 
+  /** @type {FlagEvalEVPHook | undefined} */
+  #flagEvalEVPHook
+
   /** @type {{ start: Function, stop: Function } | undefined} */
   #configurationSource
 
@@ -30,7 +34,9 @@ class FlaggingProvider extends DatadogNodeServerProvider {
       initializationTimeoutMs: config.featureFlags.DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS,
     })
 
-    this.hooks.push(new EvalMetricsHook(config))
+    if (config.DD_METRICS_OTEL_ENABLED === true) {
+      this.hooks.push(new EvalMetricsHook(config))
+    }
 
     if (config.featureFlags.DD_EXPERIMENTAL_FLAGGING_PROVIDER_SPAN_ENRICHMENT_ENABLED) {
       this.#spanEnrichmentHook = new SpanEnrichmentHook(tracer)
@@ -44,6 +50,11 @@ class FlaggingProvider extends DatadogNodeServerProvider {
     log.debug('%s created with timeout: %dms', this.constructor.name,
       config.featureFlags.DD_EXPERIMENTAL_FLAGGING_PROVIDER_INITIALIZATION_TIMEOUT_MS)
 
+    if (config.featureFlags?.DD_FLAGGING_EVALUATION_COUNTS_ENABLED !== false) {
+      this.#flagEvalEVPHook = new FlagEvalEVPHook(config)
+      this.hooks.push(this.#flagEvalEVPHook)
+    }
+
     this.#configurationSource = configurationSource.create(config, this.setConfiguration.bind(this))
     this.#configurationSource?.start()
   }
@@ -73,6 +84,8 @@ class FlaggingProvider extends DatadogNodeServerProvider {
     this.#configurationSource = undefined
     this.#spanEnrichmentHook?.destroy()
     this.#spanEnrichmentHook = undefined
+    this.#flagEvalEVPHook?.destroy()
+    this.#flagEvalEVPHook = undefined
   }
 }
 

package/packages/dd-trace/src/openfeature/writers/base.js

@@ -20,6 +20,7 @@ const EVP_ORIGIN_HEADERS = {
  * @property {number} [payloadSizeLimit] - Maximum payload size in bytes
  * @property {number} [eventSizeLimit] - Maximum individual event size in bytes
  * @property {object} [headers] - Additional HTTP headers
+ * @property {(error: Error, statusCode?: number) => string} [formatError] - Optional delivery error redaction
  */
 
 /**
@@ -80,10 +81,14 @@ function shouldSwitchFutureRoute (statusCode) {
  */
 class BaseFFEWriter {
   #destroyer
+  #formatError
   /**
    * @param {BaseFFEWriterOptions} options - Writer configuration options
    */
-  constructor ({ interval, timeout, config, endpoint, agentUrl, payloadSizeLimit, eventSizeLimit, headers }) {
+  constructor ({
+    interval, timeout, config, endpoint, agentUrl, payloadSizeLimit, eventSizeLimit, headers, formatError,
+  }) {
+    this.#formatError = formatError
     this._interval = interval ?? 1000
     this._timeout = timeout ?? 5000
 
@@ -219,10 +224,11 @@ class BaseFFEWriter {
    * @protected
    * @param {string} payload - Encoded event batch
    * @param {number} eventCount - Event count
+   * @param {(delivered: boolean) => void} [onComplete] - Final outcome after any safe fallback attempt
    */
-  _sendPayload (payload, eventCount) {
+  _sendPayload (payload, eventCount, onComplete) {
     const route = this.#createActiveRoute()
-    this.#sendRequest(payload, eventCount, route, this._fallbackRoute)
+    this.#sendRequest(payload, eventCount, route, this._fallbackRoute, onComplete)
   }
 
   /**
@@ -301,9 +307,13 @@ class BaseFFEWriter {
    * @param {number} eventCount - Event count
    * @param {ActiveWriterRoute} route - Selected route
    * @param {ActiveWriterRoute} [fallbackRoute] - Direct fallback route
+   * @param {(delivered: boolean) => void} [onComplete] - True only for a successful final response
    */
-  #sendRequest (payload, eventCount, route, fallbackRoute) {
-    request(payload, route.requestOptions, (error, response, statusCode) => {
+  #sendRequest (payload, eventCount, route, fallbackRoute, onComplete) {
+    // The request helper mutates headers. Concurrent envelopes must not share them.
+    const requestOptions = { ...route.requestOptions, headers: { ...route.requestOptions.headers } }
+    request(payload, requestOptions, (error, response, statusCode) => {
+      const errorMessage = error && (this.#formatError ? this.#formatError(error, statusCode) : error.message)
       if (fallbackRoute && isSafeToReplay(error, statusCode)) {
         log.debug(
           '%s switching from %s%s to direct intake after definitive rejection',
@@ -311,10 +321,12 @@ class BaseFFEWriter {
           route.url.href,
           route.endpoint
         )
-        this.#activateRoute(fallbackRoute)
-        this._fallbackRoute = undefined
-        route.onFallback?.()
-        this.#sendRequest(payload, eventCount, fallbackRoute)
+        if (this._requestOptions === route.requestOptions) {
+          this.#activateRoute(fallbackRoute)
+          this._fallbackRoute = undefined
+          route.onFallback?.()
+        }
+        this.#sendRequest(payload, eventCount, fallbackRoute, undefined, onComplete)
         return
       }
 
@@ -325,10 +337,13 @@ class BaseFFEWriter {
           route.url.href,
           route.endpoint
         )
-        this.#activateRoute(fallbackRoute)
-        this._fallbackRoute = undefined
-        route.onFallback?.()
-        log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, error.message)
+        if (this._requestOptions === route.requestOptions) {
+          this.#activateRoute(fallbackRoute)
+          this._fallbackRoute = undefined
+          route.onFallback?.()
+        }
+        log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, errorMessage)
+        onComplete?.(false)
         return
       }
 
@@ -340,15 +355,19 @@ class BaseFFEWriter {
           route.endpoint,
           statusCode
         )
-        this.#activateRoute(fallbackRoute)
-        this._fallbackRoute = undefined
-        route.onFallback?.()
+        if (this._requestOptions === route.requestOptions) {
+          this.#activateRoute(fallbackRoute)
+          this._fallbackRoute = undefined
+          route.onFallback?.()
+        }
         log.warn('Events request returned status %d', statusCode)
+        onComplete?.(false)
         return
       }
 
       if (
         !fallbackRoute &&
+        this._requestOptions === route.requestOptions &&
         route.onUnavailable &&
         (isSafeToReplay(error, statusCode) ||
           isTransportFailure(error, statusCode) ||
@@ -356,20 +375,22 @@ class BaseFFEWriter {
       ) {
         route.onUnavailable()
         if (error) {
-          log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, error.message)
+          log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, errorMessage)
         } else {
           log.warn('Events request returned status %d', statusCode)
         }
+        onComplete?.(false)
         return
       }
 
       if (error) {
-        log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, error.message)
+        log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, errorMessage)
       } else if (statusCode >= 200 && statusCode < 300) {
         log.debug('Successfully sent %d events', eventCount)
       } else {
         log.warn('Events request returned status %d', statusCode)
       }
+      onComplete?.(!error && statusCode >= 200 && statusCode < 300)
     })
   }
 }