dd-trace 6.7.0 → 6.8.0

package/ci/vitest-no-worker-init-setup.mjs

@@ -1,14 +1,10 @@
-import { realpathSync } from 'node:fs'
-import { relative } from 'node:path'
-import { performance } from 'node:perf_hooks'
+import { afterEach, beforeAll, beforeEach, inject } from 'vitest'
 
-import { afterEach, beforeAll, beforeEach } from 'vitest'
-
-// Instrumentation-less worker setup for DD_EXPERIMENTAL_TEST_OPT_VITEST_NO_WORKER_INIT.
-// It applies Test Optimization execution changes without initializing dd-trace in Vitest workers.
+// Instrumentation-less setup for DD_EXPERIMENTAL_TEST_OPT_VITEST_NO_WORKER_INIT.
+// It applies Test Optimization execution changes without initializing dd-trace and also supports Browser Mode.
 const VITEST_NO_WORKER_INIT_ACTIVE_ENV = 'DD_TEST_OPT_VITEST_NO_WORKER_INIT_ACTIVE'
-const isNoWorkerInitActive = getIsNoWorkerInitActive()
-const providedContext = isNoWorkerInitActive ? getProvidedContext() : {}
+const providedContext = getProvidedContext()
+const isNoWorkerInitActive = providedContext.isActive ?? getIsNoWorkerInitActive()
 const attemptToFixTests = providedContext.attemptToFixTests || {}
 const attemptToFixRetries = providedContext.attemptToFixRetries || 0
 const disabledTests = providedContext.disabledTests || {}
@@ -22,7 +18,9 @@ const isEarlyFlakeDetectionEnabled = providedContext.isEarlyFlakeDetectionEnable
 const knownTests = providedContext.knownTests || {}
 const modifiedFiles = providedContext.modifiedFiles || {}
 const quarantinedTests = providedContext.quarantinedTests || {}
-const repositoryRoot = realpath(providedContext.repositoryRoot || process.cwd())
+const isRumCorrelationEnabled = providedContext.isRumCorrelationEnabled !== false
+const rumTestExecutionIdCookieName = providedContext.rumTestExecutionIdCookieName
+const testPropertiesByFilepath = providedContext.testPropertiesByFilepath || {}
 let setVitestTaskFn
 if (isNoWorkerInitActive) {
   try {
@@ -35,7 +33,21 @@ const earlyFlakeDetectionRetriesByTask = new WeakMap()
 const earlyFlakeDetectionSkippedResults = new WeakMap()
 const earlyFlakeDetectionStartByTask = new WeakMap()
 const nextAttemptIndexByTask = new WeakMap()
-const realpaths = new Map()
+const retryAttemptIndexByTask = new WeakMap()
+const usedRumTestExecutionIds = new Set()
+let now
+let timeOrigin
+// Use an unfaked monotonic clock in Node and Vitest's parent frame in Browser Mode.
+if (typeof globalThis.process?.uptime === 'function') {
+  now = () => globalThis.process.uptime() * 1000
+  timeOrigin = Date.now() - now()
+} else {
+  const clock = globalThis.window?.parent && globalThis.window.parent !== globalThis.window
+    ? globalThis.window.parent.performance
+    : globalThis.performance
+  now = clock ? clock.now.bind(clock) : Date.now
+  timeOrigin = Number.isFinite(clock?.timeOrigin) ? clock.timeOrigin : Date.now() - now()
+}
 
 if (isNoWorkerInitActive) {
   // eslint-disable-next-line no-empty-pattern
@@ -51,11 +63,12 @@ if (isNoWorkerInitActive) {
     const isEarlyFlakeDetectionTestAttempt = isEarlyFlakeDetectionTest(testSuite, testName)
     const isQuarantinedTest = quarantinedTests[testSuite]?.[testName] && !isAttemptToFixTest
     const attemptIndex = getNextAttemptIndex(task)
+    const attemptStart = now()
     if (attemptIndex > 0) {
       recordTestOptimizationStatus(task, attemptIndex - 1)
     }
 
-    if (disabledTests[testSuite]?.[testName] && !isAttemptToFixTest) {
+    if (!isAttemptToFixTest && disabledTests[testSuite]?.[testName]) {
       skip('Skipped by Datadog Test Optimization')
     } else if (isAttemptToFixTest && attemptIndex > 0) {
       task.result.state = 'run'
@@ -66,27 +79,33 @@ if (isNoWorkerInitActive) {
       }
     }
 
-    if (isAttemptToFixTest || isEarlyFlakeDetectionTestAttempt || isQuarantinedTest) {
-      onTestFinished(() => {
-        if (attemptIndex === getFinalAttemptIndex(task)) {
-          if (isAttemptToFixTest || isEarlyFlakeDetectionTestAttempt) {
-            recordTestOptimizationStatus(task, attemptIndex, true)
-          }
-          switchQuarantinedFinalFailure(task, attemptIndex)
+    prepareRumCorrelation(task, attemptIndex)
+
+    onTestFinished(() => {
+      recordTestAttemptTiming(task, attemptIndex, attemptStart)
+      recordRetryErrorCount(task)
+      if (
+        (isAttemptToFixTest || isEarlyFlakeDetectionTestAttempt || isQuarantinedTest) &&
+        attemptIndex === getFinalAttemptIndex(task)
+      ) {
+        if (isAttemptToFixTest || isEarlyFlakeDetectionTestAttempt) {
+          recordTestOptimizationStatus(task, attemptIndex, true)
         }
-      })
-    }
+        switchQuarantinedFinalFailure(task, attemptIndex)
+      }
+      finishRumCorrelation(task, attemptIndex)
+    })
   })
 
   afterEach(function ({ task }) {
     const attemptIndex = task.meta.__ddTestOptCurrentAttemptIndex
-    if (restoreEarlyFlakeDetectionSkippedResult(task)) {
-      return
-    }
-    if (attemptIndex === getFinalAttemptIndex(task)) {
+    const restoredEarlyFlakeDetectionResult = restoreEarlyFlakeDetectionSkippedResult(task)
+    if (!restoredEarlyFlakeDetectionResult && attemptIndex === getFinalAttemptIndex(task)) {
       recordTestOptimizationStatus(task, attemptIndex)
     }
-    switchQuarantinedFinalFailure(task, attemptIndex)
+    if (!restoredEarlyFlakeDetectionResult) {
+      switchQuarantinedFinalFailure(task, attemptIndex)
+    }
   })
 }
 
@@ -110,6 +129,7 @@ function applyExecutionChanges (suite) {
       task.repeats = earlyFlakeDetectionRetries
       task.meta.__ddTestOptEfdRetries = earlyFlakeDetectionRetries
     }
+    wrapRetryCondition(task)
   }
 }
 
@@ -117,9 +137,194 @@ function getNextAttemptIndex (task) {
   const attemptIndex = nextAttemptIndexByTask.get(task) || 0
   nextAttemptIndexByTask.set(task, attemptIndex + 1)
   task.meta.__ddTestOptCurrentAttemptIndex = attemptIndex
+
+  const repeatCount = task.result?.repeatCount || 0
+  const retryAttempt = retryAttemptIndexByTask.get(task)
+  const retryAttemptIndex = retryAttempt?.repeatCount === repeatCount ? retryAttempt.index + 1 : 0
+  retryAttemptIndexByTask.set(task, {
+    index: retryAttemptIndex,
+    repeatCount,
+  })
+
   return attemptIndex
 }
 
+/**
+ * Sets the RUM correlation cookie for a sequential browser test attempt.
+ *
+ * @param {object} task
+ * @param {number} attemptIndex
+ * @returns {void}
+ */
+function prepareRumCorrelation (task, attemptIndex) {
+  // A per-origin cookie cannot identify overlapping attempts without racing.
+  if (!isRumCorrelationEnabled || isConcurrentTask(task)) return
+
+  if (
+    typeof window === 'undefined' ||
+    typeof document === 'undefined' ||
+    typeof rumTestExecutionIdCookieName !== 'string'
+  ) {
+    return
+  }
+
+  const testExecutionId = generateTestExecutionId()
+  if (
+    !testExecutionId ||
+    usedRumTestExecutionIds.has(testExecutionId) ||
+    !setRumCorrelationCookie(testExecutionId)
+  ) {
+    return
+  }
+
+  usedRumTestExecutionIds.add(testExecutionId)
+  task.meta.__ddTestOptRumTestExecutionIds ||= []
+  task.meta.__ddTestOptRumTestExecutionIds[attemptIndex] = testExecutionId
+  recordRumActivity(task, attemptIndex)
+}
+
+/**
+ * Records whether RUM was active and clears the attempt cookie.
+ *
+ * @param {object} task
+ * @param {number} attemptIndex
+ * @returns {void}
+ */
+function finishRumCorrelation (task, attemptIndex) {
+  const testExecutionId = task.meta.__ddTestOptRumTestExecutionIds?.[attemptIndex]
+  if (!testExecutionId) return
+
+  recordRumActivity(task, attemptIndex)
+  clearRumCorrelationCookie(testExecutionId)
+}
+
+/**
+ * Retains evidence that RUM was active at any point observed during the attempt.
+ *
+ * @param {object} task
+ * @param {number} attemptIndex
+ * @returns {void}
+ */
+function recordRumActivity (task, attemptIndex) {
+  if (!getIsRumActive(getRum())) return
+
+  task.meta.__ddTestOptRumActive ||= []
+  task.meta.__ddTestOptRumActive[attemptIndex] = true
+}
+
+/**
+ * Returns the RUM public API without letting application-defined accessors fail a test.
+ *
+ * @returns {object|undefined}
+ */
+function getRum () {
+  try {
+    return window.DD_RUM
+  } catch {}
+}
+
+/**
+ * Returns whether the current RUM session is active.
+ *
+ * @param {object|undefined} rum
+ * @returns {boolean}
+ */
+function getIsRumActive (rum) {
+  if (!rum) return false
+  if (typeof rum.getInternalContext !== 'function') return false
+
+  try {
+    return !!rum.getInternalContext()
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Returns whether a task or any containing suite is concurrent.
+ *
+ * @param {object} task
+ * @returns {boolean}
+ */
+function isConcurrentTask (task) {
+  let currentTask = task
+  while (currentTask) {
+    if (currentTask.concurrent === true) return true
+    currentTask = currentTask.suite
+  }
+  return false
+}
+
+/**
+ * Generates a positive 63-bit decimal test execution ID.
+ *
+ * @returns {string|undefined}
+ */
+function generateTestExecutionId () {
+  // This setup file executes in a browser even though lint validates it against the package's Node.js target.
+  // eslint-disable-next-line n/no-unsupported-features/node-builtins
+  if (typeof globalThis.crypto?.getRandomValues !== 'function') return
+
+  const words = new Uint32Array(2)
+  try {
+    // eslint-disable-next-line n/no-unsupported-features/node-builtins
+    globalThis.crypto.getRandomValues(words)
+  } catch {
+    return
+  }
+
+  words[0] &= 0x7F_FF_FF_FF
+  if (words[0] === 0 && words[1] === 0) return
+
+  return ((BigInt(words[0]) << 32n) | BigInt(words[1])).toString(10)
+}
+
+/**
+ * Sets and verifies the RUM correlation cookie for the current origin.
+ *
+ * @param {string} testExecutionId
+ * @returns {boolean}
+ */
+function setRumCorrelationCookie (testExecutionId) {
+  try {
+    // eslint-disable-next-line unicorn/no-document-cookie
+    document.cookie = `${rumTestExecutionIdCookieName}=${testExecutionId}; path=/`
+    return getRumCorrelationCookie() === testExecutionId
+  } catch {
+    return false
+  }
+}
+
+/**
+ * Clears the RUM correlation cookie if it still belongs to this attempt.
+ *
+ * @param {string} testExecutionId
+ * @returns {void}
+ */
+function clearRumCorrelationCookie (testExecutionId) {
+  try {
+    if (getRumCorrelationCookie() === testExecutionId) {
+      // eslint-disable-next-line unicorn/no-document-cookie
+      document.cookie = `${rumTestExecutionIdCookieName}=; path=/; max-age=0`
+    }
+  } catch {}
+}
+
+/**
+ * Returns the current RUM correlation cookie value.
+ *
+ * @returns {string|undefined}
+ */
+function getRumCorrelationCookie () {
+  const prefix = `${rumTestExecutionIdCookieName}=`
+  for (const cookie of document.cookie.split(';')) {
+    const normalizedCookie = cookie.trim()
+    if (normalizedCookie.startsWith(prefix)) {
+      return normalizedCookie.slice(prefix.length)
+    }
+  }
+}
+
 function recordTestOptimizationStatus (task, attemptIndex = task.result?.repeatCount || 0, onlyIfNewErrors = false) {
   const testSuite = getTestSuite(task)
   const testName = getTestName(task)
@@ -206,6 +411,76 @@ function recordManualRepeatStatus (task, attemptIndex) {
   task.meta.__ddTestOptRepeatErrorCounts[attemptIndex] = task.result?.errors?.length || 0
 }
 
+/**
+ * Records cumulative errors at the end of each configured retry attempt.
+ *
+ * @param {object} task
+ * @returns {void}
+ */
+function recordRetryErrorCount (task) {
+  const retryLimit = getRetryLimit(task)
+  if (retryLimit <= 0) return
+
+  const retryCount = task.result?.retryCount || 0
+  task.meta.__ddTestOptRetryErrorCounts ||= []
+  task.meta.__ddTestOptRetryErrorCounts[retryCount] = task.result?.errors?.length || 0
+}
+
+/**
+ * Records the wall-clock start and elapsed time for one retry or repeat execution.
+ *
+ * @param {object} task
+ * @param {number} attemptIndex
+ * @param {number} attemptStart
+ * @returns {void}
+ */
+function recordTestAttemptTiming (task, attemptIndex, attemptStart) {
+  task.meta.__ddTestOptAttemptStartTimes ||= []
+  task.meta.__ddTestOptAttemptDurations ||= []
+  task.meta.__ddTestOptAttemptStartTimes[attemptIndex] = timeOrigin + attemptStart
+  task.meta.__ddTestOptAttemptDurations[attemptIndex] = now() - attemptStart
+}
+
+/**
+ * Returns the configured retry count for numeric and object-form retry options.
+ *
+ * @param {object} task
+ * @returns {number}
+ */
+function getRetryLimit (task) {
+  return typeof task.retry === 'number' ? task.retry : task.retry?.count || 0
+}
+
+/**
+ * Wraps Vitest's retry condition so final-attempt handling follows the condition's actual result.
+ *
+ * @param {object} task
+ * @returns {void}
+ */
+function wrapRetryCondition (task) {
+  if (typeof task.retry !== 'object' || !task.retry?.condition || getRetryLimit(task) === 0) return
+
+  const condition = task.retry.condition
+  task.retry = {
+    ...task.retry,
+    condition (error) {
+      let shouldRetry = false
+      if (condition instanceof RegExp) {
+        shouldRetry = condition.test(error?.message || '')
+      } else if (typeof condition === 'function') {
+        shouldRetry = condition(error)
+      }
+
+      if (!shouldRetry && (task.result?.repeatCount || 0) >= (task.repeats || 0)) {
+        const attemptIndex = task.meta.__ddTestOptCurrentAttemptIndex
+        recordTestOptimizationStatus(task, attemptIndex)
+        markQuarantinedFailure(task)
+      }
+      return shouldRetry
+    },
+  }
+}
+
 function restoreEarlyFlakeDetectionSkippedResult (task) {
   if (!task.meta.__ddTestOptEfdSkipCurrentAttempt) {
     return false
@@ -236,7 +511,18 @@ function getFinalAttemptIndex (task) {
   if (isEarlyFlakeDetectionTest(testSuite, testName)) {
     return getEarlyFlakeDetectionRetryCountForTask(task)
   }
-  return task.repeats
+
+  const attemptIndex = task.meta.__ddTestOptCurrentAttemptIndex
+  const repeatCount = task.result?.repeatCount || 0
+  const repeatLimit = task.repeats || 0
+  const retryAttemptIndex = retryAttemptIndexByTask.get(task)?.index || 0
+  const retryLimit = getRetryLimit(task)
+  const retriesRemaining = task.result?.state === 'fail' || task.result?.state === 'run'
+    ? Math.max(retryLimit - retryAttemptIndex, 0)
+    : 0
+  const repeatsRemaining = Math.max(repeatLimit - repeatCount, 0)
+
+  return attemptIndex + retriesRemaining + (repeatsRemaining * (retryLimit + 1))
 }
 
 function switchQuarantinedFinalFailure (task, attemptIndex) {
@@ -250,9 +536,27 @@ function switchQuarantinedFinalFailure (task, attemptIndex) {
     return
   }
 
-  const retryCount = task.result?.retryCount || 0
-  const retryLimit = task.retry || 0
-  if (retryCount < retryLimit || attemptIndex < getFinalAttemptIndex(task)) {
+  if (attemptIndex < getFinalAttemptIndex(task)) {
+    return
+  }
+
+  markQuarantinedFailure(task)
+}
+
+/**
+ * Converts a quarantined failure into the passing runner state expected by Vitest.
+ *
+ * @param {object} task
+ * @returns {void}
+ */
+function markQuarantinedFailure (task) {
+  const testSuite = getTestSuite(task)
+  const testName = getTestName(task)
+  if (
+    !quarantinedTests[testSuite]?.[testName] ||
+    attemptToFixTests[testSuite]?.[testName] ||
+    task.result?.state !== 'fail'
+  ) {
     return
   }
 
@@ -292,7 +596,7 @@ function getPreviousErrorCount (errorCounts, repeatCount) {
 
 function prepareEarlyFlakeDetectionAttempt (task, attemptIndex) {
   if (attemptIndex === 0) {
-    earlyFlakeDetectionStartByTask.set(task, performance.now())
+    earlyFlakeDetectionStartByTask.set(task, now())
     return false
   }
 
@@ -308,12 +612,12 @@ function prepareEarlyFlakeDetectionAttempt (task, attemptIndex) {
   }
 
   if (attemptIndex <= retryCount) {
-    earlyFlakeDetectionStartByTask.set(task, performance.now())
+    earlyFlakeDetectionStartByTask.set(task, now())
     return false
   }
 
   if (!canReplaceVitestTaskFn()) {
-    earlyFlakeDetectionStartByTask.set(task, performance.now())
+    earlyFlakeDetectionStartByTask.set(task, now())
     return false
   }
 
@@ -334,7 +638,7 @@ function getEarlyFlakeDetectionRetryCount (task) {
   }
 
   const executionStart = earlyFlakeDetectionStartByTask.get(task)
-  const duration = executionStart === undefined ? task.result?.duration ?? 0 : performance.now() - executionStart
+  const duration = executionStart === undefined ? task.result?.duration ?? 0 : now() - executionStart
   for (const { key, limitMs } of earlyFlakeDetectionRetryThresholds) {
     if (duration < limitMs) {
       return earlyFlakeDetectionSlowRetries[key] ?? 0
@@ -376,21 +680,10 @@ function isModifiedTest (testSuite) {
 }
 
 function getTestSuite (task) {
-  let filepath = realpaths.get(task.file.filepath)
-  if (filepath === undefined) {
-    filepath = realpath(task.file.filepath)
-    realpaths.set(task.file.filepath, filepath)
-  }
-
-  return normalizePath(relative(repositoryRoot, filepath))
-}
-
-function realpath (filepath) {
-  try {
-    return realpathSync(filepath)
-  } catch {
-    return filepath
-  }
+  const filepath = task.file.filepath
+  return testPropertiesByFilepath[filepath]?.testSuite ||
+    testPropertiesByFilepath[normalizePath(filepath)]?.testSuite ||
+    normalizePath(filepath)
 }
 
 function getTestName (task) {
@@ -423,8 +716,12 @@ function getIsNoWorkerInitActive () {
 
 function getProvidedContext () {
   try {
-    return globalThis.__vitest_worker__.providedContext._ddVitestWorkerSetup || {}
+    return inject('_ddVitestWorkerSetup') || {}
   } catch {
-    return {}
+    try {
+      return globalThis.__vitest_worker__.providedContext._ddVitestWorkerSetup || {}
+    } catch {
+      return {}
+    }
   }
 }

package/index.d.ts

@@ -3734,29 +3734,74 @@ declare namespace tracer {
       flush (): void
     }
 
+    /** JSON-serializable value accepted by LLMObs Experiments. */
+    type JSONType = string | number | boolean | null | JSONType[] | { [key: string]: JSONType }
+
     /**
      * A task run over each dataset record during an experiment.
      */
-    type ExperimentTask = (input: any, config: Record<string, any>) => any | Promise<any>
+    type ExperimentTask = (
+      input: JSONType,
+      config: Record<string, JSONType>,
+      metadata?: Record<string, JSONType>
+    ) => JSONType | Promise<JSONType>
 
     /**
      * Scores a single task output. The return type selects the metric:
-     * `boolean` -> boolean, `number` -> score, anything else -> categorical.
+     * `boolean` -> boolean, `number` -> score, `string` -> categorical, anything else -> json.
+     */
+    type ExperimentEvaluator = (
+      input: JSONType,
+      output: JSONType,
+      expectedOutput: JSONType
+    ) => JSONType | Promise<JSONType>
+
+    /**
+     * Scores all rows in an experiment run and emits a summary metric.
      */
-    type ExperimentEvaluator = (input: any, output: any, expectedOutput: any) => any | Promise<any>
+    type ExperimentSummaryEvaluator = (
+      inputs: any[],
+      outputs: any[],
+      expectedOutputs: any[],
+      evaluatorResults: Record<string, any[]>,
+      metadata?: Array<Record<string, any>>
+    ) => any | Promise<any>
+
+    interface CreateDatasetOptions {
+      description?: string
+      records?: Array<{
+        id?: string,
+        inputData: JSONType,
+        expectedOutput?: JSONType,
+        metadata?: Record<string, JSONType>
+      }>
+    }
 
     interface ExperimentOptions {
       name: string
       dataset: Dataset
       task: ExperimentTask
-      /** Evaluators keyed by metric label. */
-      evaluators?: Record<string, ExperimentEvaluator>
+      /** Evaluators keyed by metric label, or named functions. */
+      evaluators?: Record<string, ExperimentEvaluator> | ExperimentEvaluator[]
+      /** Summary evaluators keyed by metric label, or named functions. */
+      summaryEvaluators?: Record<string, ExperimentSummaryEvaluator> | ExperimentSummaryEvaluator[]
       description?: string
-      config?: Record<string, any>
+      config?: Record<string, JSONType>
       tags?: Record<string, string>
     }
 
+    interface ExperimentRunOptions {
+      /** Maximum retries for task and evaluator failures. Default 0. */
+      maxRetries?: number
+      /** Delay before a retry, in milliseconds. Default 100 * (attempt + 1). */
+      retryDelay?: (attempt: number) => number
+      /** Reject on the first task/evaluator error instead of capturing it. Default false. */
+      throwOnErrors?: boolean
+    }
+
     interface PullDatasetOptions {
+      /** Dataset version to pull. Defaults to latest. */
+      version?: number
       /** Wait until at least this many records are readable (absorbs write lag). */
       expectedRecordCount?: number
       /** Maximum total time to wait, in ms. Default 30000. */
@@ -3769,19 +3814,30 @@ declare namespace tracer {
       traceId: string
       startNs: number
       durationNs: number
-      input: any
-      output: any
-      expectedOutput: any
+      input: JSONType
+      output: JSONType
+      expectedOutput: JSONType
       readonly isError: boolean
       errorType: string | null
       errorMessage: string | null
-      evaluations: Record<string, any>
+      evaluations: Record<string, JSONType>
       evaluationErrors: Record<string, string>
     }
 
+    interface ExperimentRun {
+      runId: string
+      runIteration: number
+      rows: ExperimentResultRow[]
+      summaryEvaluations: Record<string, { value: any, error: string | null }>
+    }
+
     interface ExperimentResult {
       experimentId: string
       rows: ExperimentResultRow[]
+      /** Single-run summary evaluator results. */
+      summaryEvaluations: Record<string, { value: any, error: string | null }>
+      /** Experiment runs. P0 Node experiments currently return one run. */
+      runs: ExperimentRun[]
       /** Dashboard URL for the experiment. */
       url: string
     }
@@ -3794,13 +3850,20 @@ declare namespace tracer {
     }
 
     interface Dataset {
-      addRecord (input: any, expectedOutput?: any, metadata?: Record<string, any>): Dataset
+      addRecord (input: JSONType, expectedOutput?: JSONType, metadata?: Record<string, JSONType>): Dataset
       /** Creates the dataset remotely if needed and pushes any unpushed records. */
       push (): Promise<DatasetPushResult>
       name (): string
       id (): string | null
       projectId (): string | null
-      records (): Array<{ input: any, expectedOutput: any, metadata: Record<string, any> }>
+      version (): number | null
+      latestVersion (): number | null
+      records (): Array<{
+        id: string | null,
+        input: JSONType,
+        expectedOutput: JSONType,
+        metadata: Record<string, JSONType>
+      }>
       /** Dashboard URL for the dataset, or null until pushed. */
       url (): string | null
     }
@@ -3809,12 +3872,13 @@ declare namespace tracer {
       name (): string
       experimentId (): string | null
       url (): string | null
-      run (): Promise<ExperimentResult>
+      run (options?: ExperimentRunOptions): Promise<ExperimentResult>
     }
 
     interface Experiments {
       /** Create a local dataset buffer; pushed on the first experiment run. */
       createDataset (name: string, description?: string): Dataset
+      createDataset (name: string, options?: CreateDatasetOptions): Dataset
       /** Pull an existing dataset (with records) by name. */
       pullDataset (name: string, options?: PullDatasetOptions): Promise<Dataset>
       /** Build an experiment to run over a dataset. */