dd-trace 6.9.0 → 6.11.0

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -12,13 +12,16 @@ const { getTestManagementTests: getTestManagementTestsRequest } =
   require('../test-management/get-test-management-tests')
 const { writeSettingsToCache } = require('../test-optimization-cache')
 const { CACHE_MISS, TestOptimizationHttpCache } = require('../test-optimization-http-cache')
+const { incrementCountMetric, TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION } = require('../telemetry')
 const { uploadCoverageReport: uploadCoverageReportRequest } = require('../requests/upload-coverage-report')
 const { uploadTestScreenshot: uploadTestScreenshotRequest } = require('../requests/upload-test-screenshot')
 const { parsers } = require('../../config/parsers')
 const log = require('../../log')
+const spanFormat = require('../../span_format')
 const { getSegment } = require('../../util')
 const BufferingExporter = require('../../exporters/common/buffering-exporter')
 const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
+const { TEST_STATUS } = require('../../plugins/util/test')
 const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata')
 
 const hostname = getHostname()
@@ -57,6 +60,17 @@ function getIsTestSessionTrace (trace) {
 const GIT_UPLOAD_TIMEOUT = 60_000 // 60 seconds
 const CAN_USE_CI_VIS_PROTOCOL_TIMEOUT = GIT_UPLOAD_TIMEOUT
 const MAX_COVERAGE_REPORT_FLAGS = 32
+const FINAL_FLUSH_TIMEOUT = 10_000
+const FINAL_FLUSH_FALLBACK_DELAY = 100
+
+/**
+ * @returns {Error}
+ */
+function createFinalFlushTimeoutError () {
+  const error = new Error('Timed out waiting for Test Optimization to flush')
+  error.code = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
+  return error
+}
 
 function appendLogTag (tags, key, value) {
   if (value !== undefined) {
@@ -87,6 +101,12 @@ function getLogTags (logMessage, { env, version }, gitRepositoryUrl, gitCommitSh
 }
 
 class CiVisibilityExporter extends BufferingExporter {
+  #finalFlush
+  #deferredTestSessionTraces = []
+  #pendingScreenshotUploads = new Set()
+  #screenshotFlushWaiters = new Set()
+  #deferredTestSuiteSpans = new Map()
+
   constructor (config, options = {}) {
     super(config)
     this._timer = undefined
@@ -137,18 +157,7 @@ class CiVisibilityExporter extends BufferingExporter {
       }
     })
 
-    const flush = () => {
-      if (this._writer) {
-        this._writer.flush()
-      }
-      if (this._coverageWriter) {
-        this._coverageWriter.flush()
-      }
-      if (this._logsWriter) {
-        this._logsWriter.flush()
-      }
-    }
-    globalThis[Symbol.for('dd-trace')].beforeExitHandlers.add(flush.bind(this))
+    globalThis[Symbol.for('dd-trace')].beforeExitHandlers.add(() => this.flush(() => {}))
   }
 
   shouldRequestSkippableSuites () {
@@ -399,18 +408,189 @@ class CiVisibilityExporter extends BufferingExporter {
   }
 
   export (trace) {
+    this.#resetFinalFlush()
+
+    if (this.#deferredTestSuiteSpans.size === 0) {
+      this.#exportTrace(trace)
+      return
+    }
+
+    let hasDeferredTestSuiteSpan = false
+    let immediateTrace
+    for (let index = 0; index < trace.length; index++) {
+      const formattedSpan = trace[index]
+      const spanId = formattedSpan.span_id?.toString()
+      const deferredTestSuiteSpan = this.#deferredTestSuiteSpans.get(spanId)
+      if (deferredTestSuiteSpan) {
+        if (!hasDeferredTestSuiteSpan && index > 0) immediateTrace = trace.slice(0, index)
+        hasDeferredTestSuiteSpan = true
+        if (deferredTestSuiteSpan.emitted || (this._isInitialized && !this.canReportSessionTraces())) {
+          this.#deferredTestSuiteSpans.delete(spanId)
+        } else {
+          deferredTestSuiteSpan.formattedSpan = formattedSpan
+        }
+      } else if (hasDeferredTestSuiteSpan) {
+        immediateTrace ??= []
+        immediateTrace.push(formattedSpan)
+      }
+    }
+
+    if (!hasDeferredTestSuiteSpan) this.#exportTrace(trace)
+    else if (immediateTrace) this.#exportTrace(immediateTrace)
+  }
+
+  /**
+   * Exports spans that are not retained for late updates to session, module, or suite events.
+   *
+   * @param {Array<object>} trace
+   * @returns {void}
+   */
+  #exportTrace (trace) {
     // Until it's initialized, we just store the traces as is
     if (!this._isInitialized) {
       this._traceBuffer.push(trace)
       return
     }
-    if (!this.canReportSessionTraces() && getIsTestSessionTrace(trace)) {
+    const isTestSessionTrace = getIsTestSessionTrace(trace)
+    if (!this.canReportSessionTraces() && isTestSessionTrace) {
       return
     }
-    this._export(trace)
+    if (this._export(trace, undefined, undefined, isTestSessionTrace) === false && isTestSessionTrace) {
+      this.#deferredTestSessionTraces.push(trace)
+    }
+  }
+
+  /**
+   * Retains a completed suite until finalization so a later framework error can still update it.
+   *
+   * @param {import('../../opentracing/span')} testSuiteSpan
+   * @returns {void}
+   */
+  deferTestSuiteSpan (testSuiteSpan) {
+    this.#resetFinalFlush()
+    this.#deferredTestSuiteSpans.set(testSuiteSpan.context()._spanId.toString(), {
+      span: testSuiteSpan,
+      formattedSpan: undefined,
+      emitted: false,
+    })
+  }
+
+  /**
+   * Retains formatted test suite events received from test framework workers until finalization.
+   *
+   * @param {Array<object>} trace
+   * @returns {void}
+   */
+  exportTraceWithDeferredTestSuite (trace) {
+    for (const formattedSpan of trace) {
+      if (formattedSpan.type !== 'test_suite_end') continue
+
+      this.#deferredTestSuiteSpans.set(formattedSpan.span_id.toString(), {
+        span: undefined,
+        formattedSpan,
+        emitted: false,
+      })
+    }
+    this.export(trace)
+  }
+
+  /**
+   * Applies a late framework error to every completed suite retained by the current finalization boundary.
+   *
+   * @param {Error} error
+   * @returns {void}
+   */
+  setDeferredTestSuiteError (error) {
+    for (const { span, formattedSpan, emitted } of this.#deferredTestSuiteSpans.values()) {
+      if (emitted) continue
+      if (span) {
+        span.setTag(TEST_STATUS, 'fail')
+        span.setTag('error', error)
+      } else if (formattedSpan) {
+        formattedSpan.meta[TEST_STATUS] = 'fail'
+        spanFormat.addError(formattedSpan, error)
+      }
+    }
+  }
+
+  /**
+   * Appends a completed suite while preserving the ordinary writer call shape outside finalization.
+   *
+   * @param {object} formattedSpan
+   * @param {{ deadline?: number }} [options]
+   * @returns {boolean|undefined}
+   */
+  #appendDeferredTestSuiteSpan (formattedSpan, options) {
+    const appended = options === undefined
+      ? this._writer.append([formattedSpan])
+      : this._writer.append([formattedSpan], options)
+
+    if (appended !== false && this._config.isCiVisibility) {
+      incrementCountMetric(TELEMETRY_EVENTS_ENQUEUED_FOR_SERIALIZATION)
+    }
+    return appended
+  }
+
+  /**
+   * Appends completed suites before their module and session parents are exported.
+   *
+   * @param {{ deadline?: number }} [options]
+   * @returns {void}
+   */
+  exportDeferredTestSuiteSpans (options) {
+    if (!this._writer || !this.canReportSessionTraces()) return
+
+    for (const [spanId, deferredTestSuiteSpan] of this.#deferredTestSuiteSpans) {
+      const { span, formattedSpan, emitted } = deferredTestSuiteSpan
+      if (emitted) continue
+      const updatedSpan = span && spanFormat(span)
+      if (span && formattedSpan) {
+        formattedSpan.error = updatedSpan.error
+        Object.assign(formattedSpan.meta, updatedSpan.meta)
+        Object.assign(formattedSpan.metrics, updatedSpan.metrics)
+        if (this.#appendDeferredTestSuiteSpan(formattedSpan, options) === false) continue
+        this.#deferredTestSuiteSpans.delete(spanId)
+      } else if (formattedSpan) {
+        if (this.#appendDeferredTestSuiteSpan(formattedSpan, options) === false) continue
+        this.#deferredTestSuiteSpans.delete(spanId)
+      } else {
+        if (this.#appendDeferredTestSuiteSpan(updatedSpan, options) === false) continue
+        deferredTestSuiteSpan.emitted = true
+      }
+    }
+  }
+
+  /**
+   * Discards completed suites that the selected transport cannot deliver.
+   *
+   * @returns {void}
+   */
+  resetDeferredTestSuiteSpans () {
+    this.#deferredTestSuiteSpans.clear()
+  }
+
+  /**
+   * Retries session, module, and suite traces rejected by writer backpressure within the final deadline.
+   *
+   * @param {{ deadline?: number }} options final-flush options
+   * @returns {void}
+   */
+  #exportDeferredTestSessionTraces (options) {
+    if (!this._writer || !this.canReportSessionTraces()) return
+
+    let retainedCount = 0
+
+    for (const trace of this.#deferredTestSessionTraces) {
+      if (this._writer.append(trace, options) === false) {
+        this.#deferredTestSessionTraces[retainedCount++] = trace
+      }
+    }
+    this.#deferredTestSessionTraces.length = retainedCount
   }
 
   exportCoverage (formattedCoverage) {
+    this.#resetFinalFlush()
+
     // Until it's initialized, we just store the coverages as is
     if (!this._isInitialized) {
       this._coverageBuffer.push(formattedCoverage)
@@ -455,6 +635,7 @@ class CiVisibilityExporter extends BufferingExporter {
       return
     }
 
+    this.#resetFinalFlush()
     this._export(
       this.formatLogMessage(testEnvironmentMetadata, logMessage),
       this._logsWriter,
@@ -462,32 +643,140 @@ class CiVisibilityExporter extends BufferingExporter {
     )
   }
 
-  flush (done = () => {}) {
-    if (!this._isInitialized) {
-      return done()
+  flush (done) {
+    const isFinalFlush = typeof done === 'function'
+    const onDone = done || (() => {})
+    let finalFlush
+
+    if (isFinalFlush && this.#finalFlush) {
+      if (this.#finalFlush.completed) onDone(this.#finalFlush.error)
+      else this.#finalFlush.callbacks.push(onDone)
+      return
     }
 
-    // TODO: safe to do them at once? Or do we want to do them one by one?
-    const writers = [
-      this._writer,
-      this._coverageWriter,
-      this._logsWriter,
-    ].filter(Boolean)
+    if (isFinalFlush && !this._isInitialized &&
+      this._traceBuffer.length === 0 && this._coverageBuffer.length === 0 &&
+      this.#deferredTestSuiteSpans.size === 0 && this.#deferredTestSessionTraces.length === 0 &&
+      this.#pendingScreenshotUploads.size === 0) {
+      onDone()
+      return
+    }
+
+    if (isFinalFlush) {
+      finalFlush = {
+        callbacks: [onDone],
+        completed: false,
+        error: undefined,
+      }
+      this.#finalFlush = finalFlush
+    }
 
-    let remaining = writers.length
+    const deadline = isFinalFlush ? Date.now() + FINAL_FLUSH_TIMEOUT : undefined
+    let hasCompleted = false
+    let initializationTimeoutId
+
+    const fallbackTimeoutId = isFinalFlush
+      ? setTimeout(() => {
+        complete(createFinalFlushTimeoutError())
+      }, FINAL_FLUSH_TIMEOUT + FINAL_FLUSH_FALLBACK_DELAY)
+      : undefined
+
+    const complete = (error) => {
+      if (hasCompleted) return
+      hasCompleted = true
+      clearTimeout(fallbackTimeoutId)
+      clearTimeout(initializationTimeoutId)
+      this.#screenshotFlushWaiters.delete(flushWriters)
+      if (error) log.error('Error flushing Test Optimization data', error)
+      if (!isFinalFlush) {
+        onDone(error)
+        return
+      }
 
-    if (remaining === 0) {
-      return done()
+      finalFlush.completed = true
+      finalFlush.error = error
+      const callbacks = finalFlush.callbacks
+      finalFlush.callbacks = []
+      for (const callback of callbacks) {
+        try {
+          callback(error)
+        } catch (callbackError) {
+          log.error('Error completing Test Optimization flush callback', callbackError)
+        }
+      }
     }
 
-    const onFlushComplete = () => {
-      remaining -= 1
+    const flushWriters = () => {
+      if (isFinalFlush && this.#pendingScreenshotUploads.size !== 0) {
+        this.#screenshotFlushWaiters.add(flushWriters)
+        return
+      }
+
+      const options = deadline === undefined ? undefined : { deadline }
+      if (isFinalFlush) {
+        this.exportDeferredTestSuiteSpans(options)
+        this.#exportDeferredTestSessionTraces(options)
+      }
+
+      const writers = [
+        this._writer,
+        this._coverageWriter,
+        this._logsWriter,
+      ].filter(Boolean)
+
+      let remaining = writers.length
+      let flushError
+
       if (remaining === 0) {
-        done()
+        complete()
+        return
+      }
+
+      const onFlushComplete = (error) => {
+        flushError ||= error
+        remaining -= 1
+        if (remaining === 0) complete(flushError)
       }
+
+      for (const writer of writers) writer.flush(onFlushComplete, options)
+    }
+
+    if (isFinalFlush && this._initializationRequest) {
+      const initializationRequest = this._initializationRequest
+      const { controller, options } = initializationRequest
+      initializationRequest.finalFlush = finalFlush
+      options.deadline = deadline
+      initializationTimeoutId = setTimeout(() => {
+        const error = createFinalFlushTimeoutError()
+        if (initializationRequest.finalFlush === finalFlush) controller.abort(error)
+        complete(error)
+      }, Math.max(0, deadline - Date.now()))
+    }
+
+    if (!isFinalFlush) {
+      if (this._isInitialized) flushWriters()
+      else complete()
+      return
+    }
+
+    if (this._isInitialized) {
+      flushWriters()
+      return
     }
 
-    for (const writer of writers) writer.flush(onFlushComplete)
+    this._canUseCiVisProtocolPromise.then(() => {
+      clearTimeout(initializationTimeoutId)
+      if (!hasCompleted) flushWriters()
+    })
+  }
+
+  /**
+   * Allows later test activity to establish a new finalization boundary.
+   *
+   * @returns {void}
+   */
+  #resetFinalFlush () {
+    this.#finalFlush = undefined
   }
 
   exportUncodedCoverages () {
@@ -572,13 +861,55 @@ class CiVisibilityExporter extends BufferingExporter {
    * @param {string} options.traceId - Test trace id used as the screenshot key
    * @param {string} options.idempotencyKey - Stable per-artifact key, reused on retry
    * @param {number} options.capturedAtMs - Capture time in epoch milliseconds
+   * @param {AbortSignal} [options.signal] - Additional signal used to cancel the upload
    * @param {Function} callback - Callback function (err)
    */
-  uploadTestScreenshot ({ filePath, traceId, idempotencyKey, capturedAtMs }, callback) {
+  uploadTestScreenshot ({ filePath, traceId, idempotencyKey, capturedAtMs, signal }, callback) {
     if (!this._testScreenshotUploadUrl) {
       return callback(new Error('Test screenshot upload URL not configured'))
     }
 
+    this.#resetFinalFlush()
+    const controller = new AbortController()
+    const deadline = Date.now() + FINAL_FLUSH_TIMEOUT
+    let settled = false
+    const complete = (error) => {
+      if (settled) return
+      settled = true
+      clearTimeout(timeoutId)
+      signal?.removeEventListener('abort', onAbort)
+
+      try {
+        callback(error)
+      } finally {
+        this.#pendingScreenshotUploads.delete(controller)
+        if (this.#pendingScreenshotUploads.size === 0) {
+          const waiters = [...this.#screenshotFlushWaiters]
+          this.#screenshotFlushWaiters.clear()
+          for (const waiter of waiters) queueMicrotask(waiter)
+        }
+      }
+    }
+    const onAbort = () => {
+      const error = signal.reason || Object.assign(new Error('Test screenshot upload aborted'), { code: 'ABORT_ERR' })
+      controller.abort(error)
+      complete(error)
+    }
+
+    this.#pendingScreenshotUploads.add(controller)
+    signal?.addEventListener('abort', onAbort, { once: true })
+    const timeoutId = setTimeout(() => {
+      const error = createFinalFlushTimeoutError()
+      controller.abort(error)
+      complete(error)
+    }, FINAL_FLUSH_TIMEOUT)
+    timeoutId.unref?.()
+
+    if (signal?.aborted) {
+      onAbort()
+      return
+    }
+
     uploadTestScreenshotRequest({
       filePath,
       traceId,
@@ -587,7 +918,9 @@ class CiVisibilityExporter extends BufferingExporter {
       url: this._testScreenshotUploadUrl,
       isEvpProxy: !!this._isUsingEvpProxy,
       evpProxyPrefix: this.evpProxyPrefix,
-    }, callback)
+      deadline,
+      signal: controller.signal,
+    }, complete)
   }
 }
 

package/packages/dd-trace/src/ci-visibility/exporters/request.js

@@ -0,0 +1,201 @@
+'use strict'
+
+const { Readable } = require('node:stream')
+
+const commonRequest = require('../../exporters/common/request')
+const {
+  RATE_LIMIT_MAX_WAIT_MS,
+  getMaxAttempts,
+  getRetryDelay,
+  isRetriableNetworkError,
+} = require('../../exporters/common/retry')
+const { getRateLimitResetDelay } = require('../requests/rate-limit')
+
+/**
+ * @param {AbortSignal} signal
+ * @returns {Error}
+ */
+function getAbortError (signal) {
+  return signal.reason || Object.assign(new Error('Request aborted'), { code: 'ABORT_ERR' })
+}
+
+/**
+ * Buffers a stream once so a Test Optimization exporter request can safely retry it.
+ *
+ * @param {Readable} data
+ * @param {AbortSignal} [signal]
+ * @param {(error: Error|null, data?: Buffer) => void} callback
+ * @returns {void}
+ */
+function bufferReadable (data, signal, callback) {
+  const chunks = []
+  let settled = false
+
+  const cleanup = (keepErrorListener = false) => {
+    data.removeListener('data', onData)
+    data.removeListener('end', onEnd)
+    if (!keepErrorListener) data.removeListener('error', onError)
+    signal?.removeEventListener('abort', onAbort)
+  }
+  const onData = chunk => chunks.push(chunk)
+  const onEnd = () => {
+    if (settled) return
+    settled = true
+    cleanup()
+    callback(null, Buffer.concat(chunks))
+  }
+  const onError = error => {
+    if (settled) return
+    settled = true
+    cleanup()
+    callback(error)
+  }
+  const onAbort = () => {
+    if (settled) return
+    settled = true
+    cleanup(true)
+    data.once('close', () => data.removeListener('error', onError))
+    data.destroy()
+    callback(getAbortError(signal))
+  }
+
+  data.on('data', onData)
+  data.once('end', onEnd)
+  data.once('error', onError)
+  signal?.addEventListener('abort', onAbort, { once: true })
+  if (signal?.aborted) onAbort()
+}
+
+/**
+ * Sends Test Optimization exporter data through the common single-attempt transport while
+ * keeping retry and finalization policy scoped to Test Optimization.
+ *
+ * @param {Buffer|string|Readable|Array<Buffer|string>} data
+ * @param {object} options
+ * @param {(error: Error|null, result?: string|null, statusCode?: number,
+ *   headers?: import('node:http').IncomingHttpHeaders) => void} callback
+ * @returns {void}
+ */
+function request (data, options, callback) {
+  const { signal } = options
+  if (signal?.aborted) return callback(getAbortError(signal))
+
+  if (data instanceof Readable) {
+    bufferReadable(data, signal, (error, bufferedData) => {
+      if (error) return callback(error)
+      requestBuffered(bufferedData, options, callback)
+    })
+    return
+  }
+
+  requestBuffered(data, options, callback)
+}
+
+/**
+ * Applies the Test Optimization retry policy to replayable request data.
+ *
+ * @param {Buffer|string|Array<Buffer|string>} data
+ * @param {object} options
+ * @param {(error: Error|null, result?: string|null, statusCode?: number,
+ *   headers?: import('node:http').IncomingHttpHeaders) => void} callback
+ * @returns {void}
+ */
+function requestBuffered (data, options, callback) {
+  const { signal } = options
+  const timeout = options.timeout || 2000
+  let retryTimer
+  let settled = false
+
+  const complete = (error, result, statusCode, headers) => {
+    if (settled) return
+    settled = true
+    clearTimeout(retryTimer)
+    signal?.removeEventListener('abort', onAbort)
+    callback(error, result, statusCode, headers)
+  }
+  const onAbort = () => complete(getAbortError(signal))
+
+  signal?.addEventListener('abort', onAbort, { once: true })
+  if (signal?.aborted) return onAbort()
+
+  /** @param {number} attemptIndex */
+  const attempt = attemptIndex => {
+    if (settled) return
+    if (signal?.aborted) return onAbort()
+
+    const deadline = options.deadline
+    const remaining = deadline === undefined ? Infinity : deadline - Date.now()
+    if (remaining <= 0) {
+      const error = new Error('Test Optimization request reached its finalization deadline')
+      error.code = 'ERR_DD_TEST_OPTIMIZATION_FLUSH_TIMEOUT'
+      complete(error)
+      return
+    }
+
+    if (deadline !== undefined && !commonRequest.writable) {
+      retryTimer = setTimeout(attempt, Math.min(50, remaining), attemptIndex)
+      retryTimer.unref?.()
+      return
+    }
+
+    const attemptOptions = {
+      ...options,
+      headers: options.headers ? { ...options.headers } : undefined,
+      retry: false,
+    }
+    if (deadline !== undefined) attemptOptions.timeout = Math.max(1, Math.min(timeout, remaining))
+
+    commonRequest(data, attemptOptions, (error, result, statusCode, headers) => {
+      if (settled) return
+      if (!error) {
+        complete(null, result, statusCode, headers)
+        return
+      }
+
+      const responseStatus = statusCode ?? error.status
+      const isRetriableHttpError = options.deadline !== undefined &&
+        (responseStatus === 429 || responseStatus >= 500)
+      if (options.retry === false || (attemptIndex >= getMaxAttempts(attemptOptions) &&
+        (isRetriableNetworkError(error) || isRetriableHttpError))) {
+        complete(error, result, statusCode, headers)
+        return
+      }
+
+      if (!isRetriableNetworkError(error) && !isRetriableHttpError) {
+        complete(error, result, statusCode, headers)
+        return
+      }
+
+      let retryDelay
+      if (responseStatus === 429) {
+        const resetDelay = getRateLimitResetDelay(headers)
+        if (Number.isFinite(resetDelay)) {
+          const retryRemaining = options.deadline - Date.now()
+          if (resetDelay > RATE_LIMIT_MAX_WAIT_MS || resetDelay >= retryRemaining) {
+            complete(error, result, statusCode, headers)
+            return
+          }
+          retryDelay = resetDelay
+        }
+      }
+
+      let delay = retryDelay ?? getRetryDelay(attemptOptions, attemptIndex)
+      if (options.deadline !== undefined && retryDelay === undefined) {
+        const retryRemaining = options.deadline - Date.now()
+        if (retryRemaining <= 0) {
+          complete(error, result, statusCode, headers)
+          return
+        }
+        const retryAttemptTimeout = timeout < retryRemaining ? timeout : Math.ceil(retryRemaining / 2)
+        delay = Math.min(delay, Math.max(0, retryRemaining - retryAttemptTimeout))
+      }
+
+      retryTimer = setTimeout(attempt, delay, attemptIndex + 1)
+      retryTimer.unref?.()
+    })
+  }
+
+  attempt(1)
+}
+
+module.exports = request