dd-trace 6.18.0 → 6.20.0

package/packages/dd-trace/src/debugger/redaction.js

@@ -0,0 +1,139 @@
+'use strict'
+
+// Used by both the main thread, which renders log template values, and the devtools client worker, which captures
+// snapshots and compiles probe expressions, so that both redact the same identifiers.
+
+// What a redacted value is rendered as in a log message.
+const REDACTED_PLACEHOLDER = '{redacted}'
+
+const DEFAULT_REDACTED_IDENTIFIERS = [
+  '2fa',
+  '_csrf',
+  '_csrf_token',
+  '_session',
+  '_xsrf',
+  'access_token',
+  'aiohttp_session',
+  'api_key',
+  'apisecret',
+  'apisignature',
+  'applicationkey',
+  'appkey',
+  'auth',
+  'authtoken',
+  'authorization',
+  'cc_number',
+  'certificatepin',
+  'cipher',
+  'client_secret',
+  'clientid',
+  'connect.sid',
+  'connectionstring',
+  'cookie',
+  'credentials',
+  'creditcard',
+  'csrf',
+  'csrf_token',
+  'cvv',
+  'databaseurl',
+  'db_url',
+  'encryption_key',
+  'encryptionkeyid',
+  'geo_location',
+  'gpg_key',
+  'ip_address',
+  'jti',
+  'jwt',
+  'license_key',
+  'masterkey',
+  'mysql_pwd',
+  'nonce',
+  'oauth',
+  'oauthtoken',
+  'otp',
+  'passhash',
+  'passwd',
+  'password',
+  'passwordb',
+  'pem_file',
+  'pgp_key',
+  'PHPSESSID',
+  'pin',
+  'pincode',
+  'pkcs8',
+  'private_key',
+  'publickey',
+  'pwd',
+  'recaptcha_key',
+  'refresh_token',
+  'routingnumber',
+  'salt',
+  'secret',
+  'secretKey',
+  'secrettoken',
+  'securitycode',
+  'security_answer',
+  'security_question',
+  'serviceaccountcredentials',
+  'session',
+  'sessionid',
+  'sessionkey',
+  'set_cookie',
+  'signature',
+  'signaturekey',
+  'ssh_key',
+  'ssn',
+  'symfony',
+  'token',
+  'transactionid',
+  'twilio_token',
+  'user_session',
+  'voterid',
+  'x-auth-token',
+  'x_api_key',
+  'x_csrftoken',
+  'x_forwarded_for',
+  'x_real_ip',
+  'XSRF-TOKEN',
+]
+
+module.exports = {
+  createIsRedactedIdentifier,
+  REDACTED_PLACEHOLDER,
+}
+
+/**
+ * Create a predicate for whether the value of an identifier must be redacted, based on the default list of redacted
+ * identifiers and the user's configuration.
+ *
+ * @param {object} config - The `dynamicInstrumentation` config.
+ * @param {string[]} config.DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS - Identifiers to redact in addition to the
+ *   defaults.
+ * @param {string[]} config.DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS - Identifiers to never redact.
+ * @returns {(name: string, isSymbol?: boolean) => boolean} Whether the identifier must be redacted. If `isSymbol` is
+ *   `true`, `name` is a `Symbol(...)` description.
+ */
+function createIsRedactedIdentifier ({
+  DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS: redactedIdentifiers,
+  DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS: excludedIdentifiers,
+}) {
+  const excluded = new Set(excludedIdentifiers.map((name) => normalizeName(name)))
+  const redacted = new Set()
+  for (const name of [...DEFAULT_REDACTED_IDENTIFIERS, ...redactedIdentifiers]) {
+    const normalized = normalizeName(name)
+    if (!excluded.has(normalized)) redacted.add(normalized)
+  }
+
+  return function isRedactedIdentifier (name, isSymbol) {
+    return redacted.has(normalizeName(name, isSymbol))
+  }
+}
+
+/**
+ * @param {string} name - The identifier name.
+ * @param {boolean} [isSymbol] - Whether the name is a `Symbol(...)` description.
+ */
+function normalizeName (name, isSymbol) {
+  if (isSymbol) name = name.slice(7, -1) // Remove `Symbol(` and `)`
+  return name.toLowerCase().replaceAll(/[-_@$.]/g, '')
+}

package/packages/dd-trace/src/knuth-hash.js

@@ -0,0 +1,17 @@
+'use strict'
+
+const UINT64_MODULO = 2n ** 64n
+
+// Knuth's factor for the sampling algorithm shared across Datadog tracers.
+const SAMPLING_KNUTH_FACTOR = 1_111_111_111_111_111_111n
+
+/**
+ * Hashes the lower 64 bits of a trace ID for deterministic trace sampling.
+ *
+ * @param {bigint} traceId
+ */
+function knuthHash (traceId) {
+  return (traceId * SAMPLING_KNUTH_FACTOR) % UINT64_MODULO
+}
+
+module.exports = knuthHash

package/packages/dd-trace/src/llmobs/constants/tags.js

@@ -1,5 +1,14 @@
 'use strict'
 
+const INPUT_TOKENS_METRIC_KEY = 'input_tokens'
+const OUTPUT_TOKENS_METRIC_KEY = 'output_tokens'
+const TOTAL_TOKENS_METRIC_KEY = 'total_tokens'
+const CACHE_READ_INPUT_TOKENS_METRIC_KEY = 'cache_read_input_tokens'
+const CACHE_WRITE_INPUT_TOKENS_METRIC_KEY = 'cache_write_input_tokens'
+const CACHE_WRITE_5M_INPUT_TOKENS_METRIC_KEY = 'ephemeral_5m_input_tokens'
+const CACHE_WRITE_1H_INPUT_TOKENS_METRIC_KEY = 'ephemeral_1h_input_tokens'
+const REASONING_OUTPUT_TOKENS_METRIC_KEY = 'reasoning_output_tokens'
+
 module.exports = {
   SPAN_KINDS: ['llm', 'agent', 'workflow', 'task', 'tool', 'embedding', 'retrieval', 'experiment'],
   SPAN_KIND: '_ml_obs.meta.span.kind',
@@ -42,24 +51,40 @@ module.exports = {
   MODEL_NAME: '_ml_obs.meta.model_name',
   MODEL_PROVIDER: '_ml_obs.meta.model_provider',
   UNKNOWN_MODEL_PROVIDER: 'unknown',
+  DEFAULT_MODEL: 'custom',
 
   INPUT_DOCUMENTS: '_ml_obs.meta.input.documents',
+  EXPERIMENT_INPUT: '_ml_obs.meta.input',
   INPUT_MESSAGES: '_ml_obs.meta.input.messages',
   INPUT_VALUE: '_ml_obs.meta.input.value',
   INPUT_PROMPT: '_ml_obs.meta.input.prompt',
 
   OUTPUT_DOCUMENTS: '_ml_obs.meta.output.documents',
+  EXPERIMENT_OUTPUT: '_ml_obs.meta.output',
   OUTPUT_MESSAGES: '_ml_obs.meta.output.messages',
   OUTPUT_VALUE: '_ml_obs.meta.output.value',
 
-  INPUT_TOKENS_METRIC_KEY: 'input_tokens',
-  OUTPUT_TOKENS_METRIC_KEY: 'output_tokens',
-  TOTAL_TOKENS_METRIC_KEY: 'total_tokens',
-  CACHE_READ_INPUT_TOKENS_METRIC_KEY: 'cache_read_input_tokens',
-  CACHE_WRITE_INPUT_TOKENS_METRIC_KEY: 'cache_write_input_tokens',
-  CACHE_WRITE_5M_INPUT_TOKENS_METRIC_KEY: 'ephemeral_5m_input_tokens',
-  CACHE_WRITE_1H_INPUT_TOKENS_METRIC_KEY: 'ephemeral_1h_input_tokens',
-  REASONING_OUTPUT_TOKENS_METRIC_KEY: 'reasoning_output_tokens',
+  INPUT_TOKENS_METRIC_KEY,
+  OUTPUT_TOKENS_METRIC_KEY,
+  TOTAL_TOKENS_METRIC_KEY,
+  CACHE_READ_INPUT_TOKENS_METRIC_KEY,
+  CACHE_WRITE_INPUT_TOKENS_METRIC_KEY,
+  CACHE_WRITE_5M_INPUT_TOKENS_METRIC_KEY,
+  CACHE_WRITE_1H_INPUT_TOKENS_METRIC_KEY,
+  REASONING_OUTPUT_TOKENS_METRIC_KEY,
+
+  // integrations build metric objects with these camelCase spellings. Null prototype: a metric
+  // named after an `Object.prototype` member must not resolve to an inherited property.
+  METRIC_KEY_ALIASES: Object.assign(Object.create(null), {
+    inputTokens: INPUT_TOKENS_METRIC_KEY,
+    outputTokens: OUTPUT_TOKENS_METRIC_KEY,
+    totalTokens: TOTAL_TOKENS_METRIC_KEY,
+    cacheReadTokens: CACHE_READ_INPUT_TOKENS_METRIC_KEY,
+    cacheWriteTokens: CACHE_WRITE_INPUT_TOKENS_METRIC_KEY,
+    cacheWrite5mTokens: CACHE_WRITE_5M_INPUT_TOKENS_METRIC_KEY,
+    cacheWrite1hTokens: CACHE_WRITE_1H_INPUT_TOKENS_METRIC_KEY,
+    reasoningOutputTokens: REASONING_OUTPUT_TOKENS_METRIC_KEY,
+  }),
 
   DROPPED_IO_COLLECTION_ERROR: 'dropped_io',
 

package/packages/dd-trace/src/llmobs/experiments/client.js

@@ -5,8 +5,23 @@
 
 const { Dataset, DatasetRecord } = require('./dataset')
 const { ExperimentResult } = require('./result')
+const { RemoteEvaluatorError } = require('./remote-evaluator')
 
 const API_BASE_PATH = '/api/v2/llm-obs/v1'
+const EVALUATOR_INFERENCE_API_BASE_PATH = '/api/unstable/llm-obs/v1'
+
+class ExperimentsRequestError extends Error {
+  /**
+   * @param {string} message
+   * @param {number} statusCode
+   * @param {object} responseBody
+   */
+  constructor (message, statusCode, responseBody) {
+    super(message)
+    this.statusCode = statusCode
+    this.responseBody = responseBody
+  }
+}
 
 // Control-plane host for a Datadog site, e.g.
 //   datadoghq.com        -> api.datadoghq.com
@@ -151,7 +166,15 @@ class ExperimentsClient {
 
     const text = await response.text()
     if (!response.ok) {
-      throw new Error(`${method} ${path} failed: HTTP ${response.status} ${text}`)
+      let responseBody = {}
+      try {
+        responseBody = text ? JSON.parse(text) : {}
+      } catch {}
+      throw new ExperimentsRequestError(
+        `${method} ${path} failed: HTTP ${response.status} ${text}`,
+        response.status,
+        responseBody
+      )
     }
     return text ? JSON.parse(text) : {}
   }
@@ -247,6 +270,41 @@ class ExperimentsClient {
     return experimentFromResource(this, response?.data ?? null)
   }
 
+  /**
+   * @param {string} evalName
+   * @param {object} context
+   */
+  evaluatorInfer (evalName, context) {
+    const encodedName = encodeURIComponent(evalName)
+    const path = `${EVALUATOR_INFERENCE_API_BASE_PATH}/evaluators/${encodedName}/infer`
+    return this.jsonApiRequest('POST', path, 'evaluator_inference', { context }).then(response => {
+      const attributes = response?.data?.attributes ?? {}
+      return {
+        value: attributes.value,
+        reasoning: attributes.reasoning,
+        assessment: attributes.assessment,
+        status: attributes.status,
+      }
+    }).catch(err => {
+      const attributes = err.responseBody?.data?.attributes ?? {}
+      const backendError = attributes.error
+      if (backendError !== null && typeof backendError === 'object' && !Array.isArray(backendError)) {
+        const fallback = err.statusCode === undefined ? err.message : `HTTP ${err.statusCode}`
+        throw new RemoteEvaluatorError(
+          `Remote evaluator '${evalName}' failed: ${backendError.message ?? fallback}`,
+          { status: attributes.status, backendError }
+        )
+      }
+
+      const apiError = Array.isArray(err.responseBody?.errors) ? err.responseBody.errors[0] : undefined
+      const detail = apiError !== null && typeof apiError === 'object'
+        ? apiError.detail ?? apiError.title
+        : undefined
+      const fallback = err.statusCode === undefined ? err.message : `HTTP ${err.statusCode}`
+      throw new Error(`Failed to call evaluator '${evalName}': ${detail ?? fallback}`)
+    })
+  }
+
   postExperimentEvents (experimentId, attributes) {
     return this.jsonApiRequest(
       'POST',
@@ -278,4 +336,10 @@ class ExperimentsClient {
   }
 }
 
-module.exports = { ExperimentsClient, apiHost, appHost, API_BASE_PATH }
+module.exports = {
+  ExperimentsClient,
+  apiHost,
+  appHost,
+  API_BASE_PATH,
+  EVALUATOR_INFERENCE_API_BASE_PATH,
+}

package/packages/dd-trace/src/llmobs/experiments/evaluator.js

@@ -0,0 +1,138 @@
+'use strict'
+
+const { validateEvaluatorName } = require('./util')
+
+/**
+ * @param {object} evaluator
+ * @param {string | undefined} name
+ */
+function resolveEvaluatorName (evaluator, name) {
+  const evaluatorName = name === undefined
+    ? evaluator.constructor.name
+    : typeof name === 'string' ? name.trim() : name
+  validateEvaluatorName(evaluatorName)
+  return evaluatorName
+}
+
+/**
+ * Context passed to a record-level class evaluator.
+ */
+class EvaluatorContext {
+  /**
+   * @param {{inputData: unknown, outputData: unknown, expectedOutput?: unknown, metadata?: object,
+   *   spanId?: string, traceId?: string}} options
+   */
+  constructor ({ inputData, outputData, expectedOutput, metadata = {}, spanId, traceId }) {
+    this.inputData = inputData
+    this.outputData = outputData
+    this.expectedOutput = expectedOutput
+    this.metadata = metadata
+    this.spanId = spanId
+    this.traceId = traceId
+  }
+}
+
+/**
+ * Context passed to a summary class evaluator.
+ */
+class SummaryEvaluatorContext {
+  /**
+   * @param {{inputs: unknown[], outputs: unknown[], expectedOutputs: unknown[], evaluationResults: object,
+   *   metadata?: object[]}} options
+   */
+  constructor ({ inputs, outputs, expectedOutputs, evaluationResults, metadata = [] }) {
+    this.inputs = inputs
+    this.outputs = outputs
+    this.expectedOutputs = expectedOutputs
+    this.evaluationResults = evaluationResults
+    this.metadata = metadata
+  }
+}
+
+/**
+ * A metric value with optional evaluation details.
+ */
+class EvaluatorResult {
+  /**
+   * @param {unknown} value
+   * @param {{reasoning?: string, assessment?: string, metadata?: object, tags?: object}} [options]
+   */
+  constructor (value, options = {}) {
+    this.value = value
+    this.reasoning = options.reasoning
+    this.assessment = options.assessment
+    this.metadata = options.metadata
+    this.tags = options.tags
+  }
+}
+
+/**
+ * A result that emits several named metrics from one evaluator invocation.
+ */
+class MultiEvaluatorResult {
+  /**
+   * @param {Record<string, unknown>} values
+   * @param {boolean} [prefix]
+   */
+  constructor (values, prefix = true) {
+    if (values === null || typeof values !== 'object' || Array.isArray(values)) {
+      throw new TypeError('MultiEvaluatorResult.values must be an object')
+    }
+    const keys = Object.keys(values)
+    if (keys.length === 0) throw new Error('MultiEvaluatorResult.values must be a non-empty object')
+    for (const key of keys) validateEvaluatorName(key)
+    this.values = values
+    this.prefix = prefix
+  }
+}
+
+/**
+ * Base class for reusable record-level evaluators.
+ *
+ * Subclasses may implement evaluate() synchronously or asynchronously.
+ */
+class BaseEvaluator {
+  /**
+   * @param {string} [name] Evaluator name. Defaults to the subclass name.
+   */
+  constructor (name) {
+    this.name = resolveEvaluatorName(this, name)
+  }
+
+  /**
+   * @param {EvaluatorContext} _context
+   */
+  evaluate (_context) {
+    throw new Error('BaseEvaluator subclasses must implement evaluate(context)')
+  }
+}
+
+/**
+ * Base class for reusable summary evaluators.
+ *
+ * Subclasses may implement evaluate() synchronously or asynchronously.
+ */
+class BaseSummaryEvaluator {
+  /**
+   * @param {string} [name] Evaluator name. Defaults to the subclass name.
+   */
+  constructor (name) {
+    this.name = resolveEvaluatorName(this, name)
+  }
+
+  /**
+   * @param {SummaryEvaluatorContext} _context
+   */
+  evaluate (_context) {
+    throw new Error('BaseSummaryEvaluator subclasses must implement evaluate(context)')
+  }
+}
+
+module.exports = {
+  BaseEvaluator,
+  BaseSummaryEvaluator,
+  EvaluatorContext,
+  EvaluatorResult,
+  MultiEvaluatorResult,
+  SummaryEvaluatorContext,
+}