dd-trace 5.96.0 → 5.97.0

package/packages/dd-trace/src/config/defaults.js

@@ -1,177 +1,347 @@
 'use strict'
 
-const pkg = require('../pkg')
-const { isFalse, isTrue } = require('../util')
-const { DD_MAJOR } = require('../../../../version')
-const { getEnvironmentVariable: getEnv } = require('./helper')
+const dns = require('dns')
+const util = require('util')
 
+const { DD_MAJOR } = require('../../../../version')
+const { parsers, transformers, telemetryTransformers, setWarnInvalidValue } = require('./parsers')
 const {
   supportedConfigurations,
 } = /** @type {import('./helper').SupportedConfigurationsJson} */ (require('./supported-configurations.json'))
 
-const service = getEnv('AWS_LAMBDA_FUNCTION_NAME') ||
-  getEnv('FUNCTION_NAME') || // Google Cloud Function Name set by deprecated runtimes
-  getEnv('K_SERVICE') || // Google Cloud Function Name set by newer runtimes
-  getEnv('WEBSITE_SITE_NAME') || // set by Azure Functions
-  pkg.name ||
-  'node'
+let log
+let seqId = 0
+const configWithOrigin = new Map()
+const parseErrors = new Map()
+
+if (DD_MAJOR >= 6) {
+  // Programmatic configuration of DD_IAST_SECURITY_CONTROLS_CONFIGURATION is not supported
+  // in newer major versions. This is special handled here until a better solution is found.
+  // TODO: Remove the programmatic configuration from supported-configurations.json once v5 is not supported anymore.
+  supportedConfigurations.DD_IAST_SECURITY_CONTROLS_CONFIGURATION[0].internalPropertyName =
+    supportedConfigurations.DD_IAST_SECURITY_CONTROLS_CONFIGURATION[0].configurationNames?.[0]
+  delete supportedConfigurations.DD_IAST_SECURITY_CONTROLS_CONFIGURATION[0].configurationNames
+} else {
+  // Default value for DD_TRACE_STARTUP_LOGS is 'false' in older major versions.
+  // This is special handled here until a better solution is found.
+  // TODO: Remove this here once v5 is not supported anymore.
+  supportedConfigurations.DD_TRACE_STARTUP_LOGS[0].default = 'false'
+}
 
 /**
- * @param {string|null} raw
- * @param {string} type
- * @returns {string|number|boolean|Record<string, string>|unknown[]|undefined}
+ * Warns about an invalid value for an option and adds the error to the last telemetry entry if it is not already set.
+ * Logging happens only if the error is not already set or the option name is different from the last telemetry entry.
+ *
+ * @param {unknown} value - The value that is invalid.
+ * @param {string} optionName - The name of the option.
+ * @param {string} source - The source of the value.
+ * @param {string} baseMessage - The base message to use for the warning.
+ * @param {Error} [error] - An error that was thrown while parsing the value.
  */
-function parseDefaultByType (raw, type) {
-  if (raw === null) {
-    return
+function warnInvalidValue (value, optionName, source, baseMessage, error) {
+  const canonicalName = (optionsTable[optionName]?.canonicalName ?? optionName) + source
+  // Lazy load log module to avoid circular dependency
+  if (!parseErrors.has(canonicalName)) {
+    // TODO: Rephrase: It will fallback to former source (or default if not set)
+    let message = `${baseMessage}: ${util.inspect(value)} for ${optionName} (source: ${source}), picked default`
+    if (error) {
+      error.stack = error.toString()
+      message += `\n\n${util.inspect(error)}`
+    }
+    parseErrors.set(canonicalName, { message })
+    log ??= require('../log')
+    const logLevel = error ? 'error' : 'warn'
+    log[logLevel](message)
   }
+}
+setWarnInvalidValue(warnInvalidValue)
 
-  switch (type) {
-    case 'boolean':
-      if (isTrue(raw)) return true
-      if (isFalse(raw)) return false
-      // TODO: What should we do with these?
-      return
-    case 'int':
-    case 'decimal': {
-      return Number(raw)
-    }
-    case 'array': {
-      if (!raw || raw.length === 0) return []
-      // TODO: Make the parsing a helper that is reused.
-      return raw.split(',').map(item => {
-        const colonIndex = item.indexOf(':')
-        if (colonIndex === -1) {
-          return item.trim()
-        }
-        const key = item.slice(0, colonIndex).trim()
-        const value = item.slice(colonIndex + 1).trim()
-        return `${key}:${value}`
-      })
+/** @type {import('./config-types').ConfigDefaults} */
+const defaults = {
+  instrumentationSource: 'manual',
+  isServiceUserProvided: false,
+  isServiceNameInferred: true,
+  plugins: true,
+  isCiVisibility: false,
+  lookup: dns.lookup,
+  logger: undefined,
+}
+
+for (const [name, value] of Object.entries(defaults)) {
+  configWithOrigin.set(`${name}default`, {
+    name,
+    value: value ?? null,
+    origin: 'default',
+    seq_id: seqId++,
+  })
+}
+
+/**
+ * @param {unknown} value
+ * @param {string} origin
+ * @param {string} optionName
+ */
+function generateTelemetry (value = null, origin, optionName) {
+  const { type, canonicalName = optionName } = configurationsTable[optionName] ?? { type: typeof value }
+  // TODO: Consider adding a preParser hook to the parsers object.
+  if (canonicalName === 'OTEL_RESOURCE_ATTRIBUTES') {
+    value = telemetryTransformers.MAP(value)
+  }
+  // TODO: Should we not send defaults to telemetry to reduce size?
+  // TODO: How to handle aliases/actual names in the future? Optional fields? Normalize the name at intake?
+  // TODO: Validate that space separated tags are parsed by the backend. Optimizations would be possible with that.
+  // TODO: How to handle telemetry reporting for aliases?
+  if (value !== null) {
+    if (telemetryTransformers[type]) {
+      value = telemetryTransformers[type](value)
+    } else if (typeof value === 'object' && value !== null) {
+      value = value instanceof URL
+        ? String(value)
+        : JSON.stringify(value)
+    } else if (typeof value === 'function') {
+      value = value.name || 'function'
     }
-    case 'map': {
-      if (!raw || raw.length === 0) return {}
-      // TODO: Make the parsing a helper that is reused.
-      /** @type {Record<string, string>} */
-      const entries = {}
-      for (const item of raw.split(',')) {
-        const colonIndex = item.indexOf(':')
-        if (colonIndex === -1) {
-          const key = item.trim()
-          if (key.length > 0) {
-            entries[key] = ''
+  }
+  const telemetryEntry = {
+    name: canonicalName,
+    value,
+    origin,
+    seq_id: seqId++,
+  }
+  const error = parseErrors.get(`${canonicalName}${origin}`)
+  if (error) {
+    parseErrors.delete(`${canonicalName}${origin}`)
+    telemetryEntry.error = error
+  }
+  configWithOrigin.set(`${canonicalName}${origin}`, telemetryEntry)
+}
+
+// Iterate over the object and always handle the leaf properties as lookup.
+// Example entries:
+//
+// cloudPayloadTagging: {
+//   nestedProperties: [
+//     'rules',
+//     'requestsEnabled',
+//     'responses',
+//   ],
+//   option: {
+//     property: 'rules',
+//     parser: parsers.JSON,
+//     canonicalName: 'DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING',
+//     transformer: transformers.toCamelCase,
+//   },
+// },
+// 'cloudPayloadTagging.responses': {
+//   nestedProperties: [
+//     'enabled',
+//   ],
+// },
+// 'cloudPayloadTagging.rules': {},
+// 'cloudPayloadTagging.requestsEnabled': {},
+// 'cloudPayloadTagging.responses.enabled': {}
+const optionsTable = {
+  // Additional properties that are not supported by the supported-configurations.json file.
+  lookup: {
+    transformer (value) {
+      if (typeof value === 'function') {
+        return value
+      }
+    },
+    property: 'lookup',
+  },
+  logger: {
+    transformer (object) {
+      // Create lazily to avoid the overhead when not used.
+      // Match at least one log level.
+      const knownLogLevels = new Set(supportedConfigurations.DD_TRACE_LOG_LEVEL[0].allowed?.split('|'))
+      if (typeof object !== 'object' || object === null) {
+        return object
+      }
+      let matched = false
+      for (const logLevel of knownLogLevels) {
+        if (object[logLevel] !== undefined) {
+          if (typeof object[logLevel] !== 'function') {
+            warnInvalidValue(object[logLevel], 'logger', 'default', `Invalid log level ${logLevel}`)
+            return
           }
-          continue
-        }
-        const key = item.slice(0, colonIndex).trim()
-        const value = item.slice(colonIndex + 1).trim()
-        if (key.length > 0) {
-          entries[key] = value
+          matched = true
         }
       }
-      return entries
-    }
-    default:
-      return raw
+      if (matched) {
+        return object
+      }
+    },
+    property: 'logger',
+  },
+  isCiVisibility: {
+    property: 'isCiVisibility',
+  },
+  plugins: {
+    property: 'plugins',
+  },
+}
+
+const parser = (value, optionName, source) => {
+  const { type, canonicalName = optionName } = configurationsTable[optionName]
+  const parsed = parsers[type](value, canonicalName)
+  if (parsed === undefined) {
+    warnInvalidValue(value, optionName, source, `Invalid ${type} input`)
   }
+  return parsed
 }
 
-/** @type {Record<string, unknown>} */
-const metadataDefaults = {}
-for (const entries of Object.values(supportedConfigurations)) {
+/**
+ * @template {import('./config-types').ConfigPath} TPath
+ * @type {Partial<Record<TPath, {
+ *   property?: string,
+ *   parser: (value: unknown, optionName: string, source: string) => unknown,
+ *   canonicalName?: string,
+ *   transformer?: (value: unknown, optionName: string, source: string) => unknown,
+ *   telemetryTransformer?: (value: unknown) => unknown
+ * }>>} ConfigurationsTable
+ */
+const configurationsTable = {}
+
+// One way aliases. Must be applied in apply calculated entries.
+const fallbackConfigurations = new Map()
+
+const regExps = {}
+
+for (const [canonicalName, entries] of Object.entries(supportedConfigurations)) {
+  if (entries.length !== 1) {
+    // TODO: Determine if we really want to support multiple entries for a canonical name.
+    // This would be needed to show official support for multiple diverging implementations
+    // at a time with by checking for another configuration that is not the canonical name.
+    throw new Error(
+      `Multiple entries found for canonical name: ${canonicalName}. ` +
+      'This is currently not supported and must be implemented, if needed.'
+    )
+  }
   for (const entry of entries) {
-    // TODO: Replace $dynamic with method names that would be called and that
-    // are also called when the user passes through the value. That way the
-    // handling is unified and methods can be declared as default.
-    // The name of that method should be expressive for users.
-    // TODO: Add handling for all environment variable names. They should not
-    // need a configuration name for being listed with their default.
-    if (!Array.isArray(entry.configurationNames)) {
-      continue
+    const configurationNames = entry.internalPropertyName ? [entry.internalPropertyName] : entry.configurationNames
+    const fullPropertyName = configurationNames?.[0] ?? canonicalName
+    const type = entry.type.toUpperCase()
+
+    let transformer = transformers[entry.transform]
+    if (entry.allowed) {
+      regExps[entry.allowed] ??= new RegExp(`^(${entry.allowed})$`, 'i')
+      const allowed = regExps[entry.allowed]
+      const originalTransform = transformer
+      transformer = (value, optionName, source) => {
+        if (!allowed.test(value)) {
+          warnInvalidValue(value, optionName, source, 'Invalid value')
+          return
+        }
+        if (originalTransform) {
+          value = originalTransform(value)
+        }
+        return value
+      }
     }
 
-    const parsedValue = parseDefaultByType(entry.default, entry.type)
-    for (const configurationName of entry.configurationNames) {
-      metadataDefaults[configurationName] = entry.default === null ? undefined : parsedValue
+    const option = { parser, type }
+
+    if (fullPropertyName !== canonicalName) {
+      option.property = fullPropertyName
+      option.canonicalName = canonicalName
+      configurationsTable[fullPropertyName] = option
+    }
+    if (transformer) {
+      option.transformer = transformer
+    }
+    if (entry.configurationNames) {
+      addOption(option, type, entry.configurationNames)
+    }
+    configurationsTable[canonicalName] = option
+
+    if (entry.default === null) {
+      defaults[fullPropertyName] = undefined
+    } else {
+      let parsedDefault = parser(entry.default, fullPropertyName, 'default')
+      if (entry.transform) {
+        parsedDefault = transformer(parsedDefault, fullPropertyName, 'default')
+      }
+      defaults[fullPropertyName] = parsedDefault
+    }
+    generateTelemetry(defaults[fullPropertyName], 'default', fullPropertyName)
+
+    if (entry.aliases) {
+      for (const alias of entry.aliases) {
+        if (!supportedConfigurations[alias]) {
+          // An actual alias has no matching entry
+          continue
+        }
+        if (!supportedConfigurations[alias].aliases?.includes(canonicalName)) {
+          // Alias will be replaced with the full property name of the alias, if it exists.
+          fallbackConfigurations.set(fullPropertyName, alias)
+        }
+      }
     }
   }
 }
 
-// Defaults required by JS config merge/applyCalculated that are not represented in supported-configurations.
-const defaultsWithoutSupportedConfigurationEntry = {
-  'cloudPayloadTagging.rules': [],
-  'cloudPayloadTagging.requestsEnabled': false,
-  'cloudPayloadTagging.responsesEnabled': false,
-  isAzureFunction: false,
-  isCiVisibility: false,
-  isGCPFunction: false,
-  instrumentationSource: 'manual',
-  isServiceUserProvided: false,
-  isServiceNameInferred: true,
-  lookup: undefined,
-  plugins: true,
+// Replace the alias with the canonical property name.
+for (const [fullPropertyName, alias] of fallbackConfigurations) {
+  if (configurationsTable[alias].property) {
+    fallbackConfigurations.set(fullPropertyName, configurationsTable[alias].property)
+  }
 }
 
-// These values are documented in supported-configurations as CI Visibility
-// defaults. Keep startup baseline false and let #applyCalculated() switch them
-// when CI Visibility is active.
-// TODO: These entries should be removed. They are off by default
-// because they rely on other configs.
-const defaultsWithConditionalRuntimeBehavior = {
-  startupLogs: DD_MAJOR >= 6,
-  isGitUploadEnabled: false,
-  isImpactedTestsEnabled: false,
-  isIntelligentTestRunnerEnabled: false,
-  isManualApiEnabled: false,
-  isTestManagementEnabled: false,
-  // TODO: These are not conditional, they would just be of type number.
-  'dogstatsd.port': '8125',
-  port: '8126',
-  // Override due to expecting numbers, not strings. TODO: Replace later.
-  'grpc.client.error.statuses': [
-    1,
-    2,
-    3,
-    4,
-    5,
-    6,
-    7,
-    8,
-    9,
-    10,
-    11,
-    12,
-    13,
-    14,
-    15,
-    16,
-  ],
-  'grpc.server.error.statuses': [
-    2,
-    3,
-    4,
-    5,
-    6,
-    7,
-    8,
-    9,
-    10,
-    11,
-    12,
-    13,
-    14,
-    15,
-    16,
-  ],
-}
+function addOption (option, type, configurationNames) {
+  for (const name of configurationNames) {
+    let index = -1
+    let lastNestedProperties
+    while (true) {
+      const nextIndex = name.indexOf('.', index + 1)
+      const intermediateName = nextIndex === -1 ? name : name.slice(0, nextIndex)
+      if (lastNestedProperties) {
+        lastNestedProperties.add(intermediateName.slice(index + 1))
+      }
 
-/** @type {Record<string, unknown>} */
-const defaults = {
-  ...defaultsWithoutSupportedConfigurationEntry,
-  ...metadataDefaults,
-  ...defaultsWithConditionalRuntimeBehavior,
-  service,
-  version: pkg.version,
+      if (nextIndex === -1) {
+        if (optionsTable[name]) {
+          if (optionsTable[name].nestedProperties && !optionsTable[name].option) {
+            optionsTable[name].option = option
+            break
+          }
+          throw new Error(`Duplicate configuration name: ${name}`)
+        }
+        optionsTable[name] = option
+        break
+      }
+
+      lastNestedProperties = new Set()
+      index = nextIndex
+
+      if (!optionsTable[intermediateName]) {
+        optionsTable[intermediateName] = {
+          nestedProperties: lastNestedProperties,
+        }
+      } else if (optionsTable[intermediateName].nestedProperties) {
+        lastNestedProperties = optionsTable[intermediateName].nestedProperties
+      } else {
+        optionsTable[intermediateName] = {
+          nestedProperties: lastNestedProperties,
+          option: optionsTable[intermediateName],
+        }
+      }
+    }
+  }
 }
 
-module.exports = defaults
+module.exports = {
+  configurationsTable,
+
+  defaults,
+
+  fallbackConfigurations,
+
+  optionsTable,
+
+  configWithOrigin,
+
+  parseErrors,
+
+  generateTelemetry,
+}

package/packages/dd-trace/src/config/generated-config-types.d.ts

@@ -88,10 +88,12 @@ export interface GeneratedConfig {
   DD_CIVISIBILITY_TEST_MODULE_ID: string | undefined;
   DD_CIVISIBILITY_TEST_SESSION_ID: string | undefined;
   DD_CUSTOM_TRACE_ID: string | undefined;
+  DD_ENABLE_LAGE_PACKAGE_NAME: boolean;
   DD_ENABLE_NX_SERVICE_NAME: boolean;
   DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_DIR: string;
   DD_EXPERIMENTAL_TEST_OPT_GIT_CACHE_ENABLED: boolean;
   DD_EXPERIMENTAL_TEST_OPT_SETTINGS_CACHE: string;
+  DD_EXPERIMENTAL_TEST_REQUESTS_FS_CACHE: boolean;
   DD_EXTERNAL_ENV: string | undefined;
   DD_GIT_BRANCH: string | undefined;
   DD_GIT_COMMIT_AUTHOR_DATE: string | undefined;
@@ -379,6 +381,7 @@ export interface GeneratedConfig {
   env: string | undefined;
   experimental: {
     aiguard: {
+      block: boolean;
       enabled: boolean;
       endpoint: string | undefined;
       maxContentSize: number;
@@ -475,7 +478,6 @@ export interface GeneratedConfig {
   };
   openAiLogsEnabled: boolean;
   OTEL_EXPORTER_OTLP_ENDPOINT: string | undefined;
-  OTEL_LOG_LEVEL: "debug" | "info" | "warn" | "error" | undefined;
   OTEL_LOGS_EXPORTER: "none" | "otlp" | undefined;
   OTEL_METRICS_EXPORTER: "none" | "otlp" | undefined;
   OTEL_RESOURCE_ATTRIBUTES: Record<string, string>;
@@ -547,6 +549,7 @@ export interface GeneratedConfig {
     debug: boolean;
     dependencyCollection: boolean;
     enabled: boolean;
+    extendedHeartbeatInterval: number;
     heartbeatInterval: number;
     logCollection: boolean;
     metrics: boolean;

package/packages/dd-trace/src/config/helper.js

@@ -9,6 +9,9 @@
  * @property {string|number|boolean|null|object|unknown[]} default
  * @property {string[]} [aliases]
  * @property {string[]} [configurationNames]
+ * @property {string} [internalPropertyName]
+ * @property {string} [transform]
+ * @property {string} [allowed]
  * @property {string|boolean} [deprecated]
  */
 
@@ -57,6 +60,13 @@ for (const [canonical, configuration] of Object.entries(supportedConfigurations)
 const aliasToCanonical = {}
 for (const canonical of Object.keys(aliases)) {
   for (const alias of aliases[canonical]) {
+    if (supportedConfigurations[alias]) {
+      // Allow 'fallback' aliases to be used for other configurations.
+      // This is used to handle the case where an alias could be used for multiple configurations.
+      // For example, OTEL_EXPORTER_OTLP_ENDPOINT is used for OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
+      // and OTEL_EXPORTER_OTLP_METRICS_ENDPOINT.
+      continue
+    }
     if (aliasToCanonical[alias]) {
       throw new Error(`The alias ${alias} is already used for ${aliasToCanonical[alias]}.`)
     }
@@ -99,22 +109,37 @@ function loadStableConfig () {
 }
 
 function getValueFromSource (name, source) {
-  const value = source[name]
+  if (source[name] !== undefined) {
+    return source[name]
+  }
 
-  if (value === undefined && aliases[name]) {
+  if (aliases[name]) {
     for (const alias of aliases[name]) {
       if (source[alias] !== undefined) {
         return source[alias]
       }
     }
   }
+}
 
-  return value
+function getEnvNameFromSource (name, source) {
+  if (source[name] !== undefined) {
+    return name
+  }
+
+  if (aliases[name]) {
+    for (const alias of aliases[name]) {
+      if (source[alias] !== undefined) {
+        return alias
+      }
+    }
+  }
 }
 
 function validateAccess (name) {
-  if ((name.startsWith('DD_') || name.startsWith('OTEL_') || aliasToCanonical[name]) &&
-    !supportedConfigurations[name]) {
+  if ((name.startsWith('DD_') || name.startsWith('OTEL_')) &&
+    !supportedConfigurations[name] &&
+    !aliasToCanonical[name]) {
     throw new Error(`Missing ${name} env/configuration in "supported-configurations.json" file.`)
   }
 }
@@ -144,10 +169,9 @@ module.exports = {
    *
    * @returns {TracerEnv} The environment variables
    */
-  getEnvironmentVariables () {
+  getEnvironmentVariables (source = process.env, internalOnly = false) {
     const configs = {}
-    for (const [key, value] of Object.entries(process.env)) {
-      // TODO(BridgeAR): Handle telemetry reporting for aliases.
+    for (const [key, value] of Object.entries(source)) {
       if (key.startsWith('DD_') || key.startsWith('OTEL_') || aliasToCanonical[key]) {
         if (supportedConfigurations[key]) {
           configs[key] = value
@@ -155,7 +179,7 @@ module.exports = {
           // The alias should only be used if the actual configuration is not set
           // In case that more than a single alias exist, use the one defined first in our own order
           for (const alias of aliases[aliasToCanonical[key]]) {
-            if (process.env[alias] !== undefined) {
+            if (source[alias] !== undefined) {
               configs[aliasToCanonical[key]] = value
               break
             }
@@ -165,9 +189,10 @@ module.exports = {
         //   debug(
         //     `Missing configuration ${env} in supported-configurations file. The environment variable is ignored.`
         //   )
+        // This could be moved inside the main config logic.
         }
         deprecationMethods[key]?.()
-      } else {
+      } else if (!internalOnly) {
         configs[key] = value
       }
     }
@@ -211,4 +236,28 @@ module.exports = {
       return getValueFromSource(name, localStableConfig)
     }
   },
+
+  /**
+   * Returns the actual environment variable name used for a supported configuration
+   * from a specific environment-based source.
+   *
+   * @param {string} name Environment variable name
+   * @returns {string|undefined}
+   */
+  getConfiguredEnvName (name) {
+    validateAccess(name)
+
+    if (!stableConfigLoaded) {
+      loadStableConfig()
+    }
+
+    for (const source of [fleetStableConfig, process.env, localStableConfig]) {
+      if (source !== undefined) {
+        const fromSource = getEnvNameFromSource(name, source)
+        if (fromSource !== undefined) {
+          return fromSource
+        }
+      }
+    }
+  },
 }