dd-trace 5.13.0 → 5.14.0

package/README.md

@@ -108,4 +108,11 @@ If you would like to trace your bundled application then please read this page o
 
 ## Security Vulnerabilities
 
-Please refer to the [SECURITY.md](https://github.com/DataDog/dd-trace-js/blob/master/SECURITY.md) document if you have found a security issue.
+Please refer to the [SECURITY.md](https://github.com/DataDog/dd-trace-js/blob/master/SECURITY.md) document if you have found a security issue.
+
+## Datadog With OpenTelemetery
+
+Please refer to the [Node.js Custom Instrumentation using OpenTelemetry API](https://docs.datadoghq.com/tracing/trace_collection/custom_instrumentation/nodejs/otel/) document. It includes information on how to use the OpenTelemetry API with dd-trace-js
+
+Note that our internal implementation of the OpenTelemetry API is currently set within the version range `>=1.0.0 <1.9.0`. This range will be updated at a regular cadence therefore, we recommend updating your tracer to the latest release to ensure up to date support.
+

package/index.d.ts

@@ -144,6 +144,7 @@ interface Plugins {
   "aws-sdk": tracer.plugins.aws_sdk;
   "bunyan": tracer.plugins.bunyan;
   "cassandra-driver": tracer.plugins.cassandra_driver;
+  "child_process": tracer.plugins.child_process;
   "connect": tracer.plugins.connect;
   "couchbase": tracer.plugins.couchbase;
   "cucumber": tracer.plugins.cucumber;
@@ -1207,6 +1208,12 @@ declare namespace tracer {
      */
     interface cassandra_driver extends Instrumentation {}
 
+    /**
+     * This plugin automatically instruments the
+     * [child_process](https://nodejs.org/api/child_process.html) module.
+     */
+    interface child_process extends Instrumentation {}
+
     /**
      * This plugin automatically instruments the
      * [connect](https://github.com/senchalabs/connect) module.

package/init.js

@@ -1,7 +1,31 @@
 'use strict'
 
-const tracer = require('.')
+const path = require('path')
+const Module = require('module')
 
-tracer.init()
+let initBailout = false
 
-module.exports = tracer
+if (process.env.DD_INJECTION_ENABLED) {
+  // If we're running via single-step install, and we're not in the app's
+  // node_modules, then we should not initialize the tracer. This prevents
+  // single-step-installed tracer from clobbering the manually-installed tracer.
+  let resolvedInApp
+  const entrypoint = process.argv[1]
+  try {
+    resolvedInApp = Module.createRequire(entrypoint).resolve('dd-trace')
+  } catch (e) {
+    // Ignore. If we can't resolve the module, we assume it's not in the app.
+  }
+  if (resolvedInApp) {
+    const ourselves = path.join(__dirname, 'index.js')
+    if (ourselves !== resolvedInApp) {
+      initBailout = true
+    }
+  }
+}
+
+if (!initBailout) {
+  const tracer = require('.')
+  tracer.init()
+  module.exports = tracer
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dd-trace",
-  "version": "5.13.0",
+  "version": "5.14.0",
   "description": "Datadog APM tracing client for JavaScript",
   "main": "index.js",
   "typings": "index.d.ts",
@@ -74,9 +74,9 @@
     "@datadog/native-iast-rewriter": "2.3.1",
     "@datadog/native-iast-taint-tracking": "2.1.0",
     "@datadog/native-metrics": "^2.0.0",
-    "@datadog/pprof": "5.2.0",
+    "@datadog/pprof": "5.3.0",
     "@datadog/sketches-js": "^2.1.0",
-    "@opentelemetry/api": "^1.0.0",
+    "@opentelemetry/api": ">=1.0.0 <1.9.0",
     "@opentelemetry/core": "^1.14.0",
     "crypto-randomuuid": "^1.0.0",
     "dc-polyfill": "^0.1.4",

package/packages/dd-trace/src/appsec/iast/iast-log.js

@@ -2,35 +2,9 @@
 
 const dc = require('dc-polyfill')
 const log = require('../../log')
-const { calculateDDBasePath } = require('../../util')
 
 const telemetryLog = dc.channel('datadog:telemetry:log')
 
-const ddBasePath = calculateDDBasePath(__dirname)
-const EOL = '\n'
-const STACK_FRAME_LINE_REGEX = /^\s*at\s/gm
-
-function sanitize (logEntry, stack) {
-  if (!stack) return logEntry
-
-  let stackLines = stack.split(EOL)
-
-  const firstIndex = stackLines.findIndex(l => l.match(STACK_FRAME_LINE_REGEX))
-
-  const isDDCode = firstIndex > -1 && stackLines[firstIndex].includes(ddBasePath)
-  stackLines = stackLines
-    .filter((line, index) => (isDDCode && index < firstIndex) || line.includes(ddBasePath))
-    .map(line => line.replace(ddBasePath, ''))
-
-  logEntry.stack_trace = stackLines.join(EOL)
-
-  if (!isDDCode) {
-    logEntry.message = 'omitted'
-  }
-
-  return logEntry
-}
-
 function getTelemetryLog (data, level) {
   try {
     data = typeof data === 'function' ? data() : data
@@ -42,18 +16,13 @@ function getTelemetryLog (data, level) {
       message = String(data.message || data)
     }
 
-    let logEntry = {
+    const logEntry = {
       message,
       level
     }
-
     if (data.stack) {
-      logEntry = sanitize(logEntry, data.stack)
-      if (logEntry.stack_trace === '') {
-        return
-      }
+      logEntry.stack_trace = data.stack
     }
-
     return logEntry
   } catch (e) {
     log.error(e)

package/packages/dd-trace/src/config.js

@@ -16,7 +16,7 @@ const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('./plugins/util/tags')
 const { getGitMetadataFromGitProperties, removeUserSensitiveInfo } = require('./git_properties')
 const { updateConfig } = require('./telemetry')
 const telemetryMetrics = require('./telemetry/metrics')
-const { getIsGCPFunction, getIsAzureFunctionConsumptionPlan } = require('./serverless')
+const { getIsGCPFunction, getIsAzureFunction } = require('./serverless')
 const { ORIGIN_KEY } = require('./constants')
 
 const tracerMetrics = telemetryMetrics.manager.namespace('tracers')
@@ -339,7 +339,7 @@ class Config {
 
     // Requires an accompanying DD_APM_OBFUSCATION_MEMCACHED_KEEP_COMMAND=true in the agent
     this.memcachedCommandEnabled = isTrue(DD_TRACE_MEMCACHED_COMMAND_ENABLED)
-    this.isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan()
+    this.isAzureFunction = getIsAzureFunction()
     this.spanLeakDebug = Number(DD_TRACE_SPAN_LEAK_DEBUG)
     this.installSignature = {
       id: DD_INSTRUMENTATION_INSTALL_ID,
@@ -417,8 +417,8 @@ class Config {
   _isInServerlessEnvironment () {
     const inAWSLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined
     const isGCPFunction = getIsGCPFunction()
-    const isAzureFunctionConsumptionPlan = getIsAzureFunctionConsumptionPlan()
-    return inAWSLambda || isGCPFunction || isAzureFunctionConsumptionPlan
+    const isAzureFunction = getIsAzureFunction()
+    return inAWSLambda || isGCPFunction || isAzureFunction
   }
 
   // for _merge to work, every config value must have a default value
@@ -549,6 +549,7 @@ class Config {
       DD_IAST_REDACTION_VALUE_PATTERN,
       DD_IAST_REQUEST_SAMPLING,
       DD_IAST_TELEMETRY_VERBOSITY,
+      DD_INJECTION_ENABLED,
       DD_INSTRUMENTATION_TELEMETRY_ENABLED,
       DD_INSTRUMENTATION_CONFIG_ID,
       DD_LOGS_INJECTION,
@@ -705,7 +706,14 @@ class Config {
     this._setBoolean(env, 'telemetry.debug', DD_TELEMETRY_DEBUG)
     this._setBoolean(env, 'telemetry.dependencyCollection', DD_TELEMETRY_DEPENDENCY_COLLECTION_ENABLED)
     this._setValue(env, 'telemetry.heartbeatInterval', maybeInt(Math.floor(DD_TELEMETRY_HEARTBEAT_INTERVAL * 1000)))
-    this._setBoolean(env, 'telemetry.logCollection', coalesce(DD_TELEMETRY_LOG_COLLECTION_ENABLED, DD_IAST_ENABLED))
+    const hasTelemetryLogsUsingFeatures =
+      isTrue(DD_IAST_ENABLED) ||
+      isTrue(DD_PROFILING_ENABLED) ||
+      (typeof DD_INJECTION_ENABLED === 'string' && DD_INJECTION_ENABLED.split(',').includes('profiling'))
+        ? true
+        : undefined
+    this._setBoolean(env, 'telemetry.logCollection', coalesce(DD_TELEMETRY_LOG_COLLECTION_ENABLED,
+      hasTelemetryLogsUsingFeatures))
     this._setBoolean(env, 'telemetry.metrics', DD_TELEMETRY_METRICS_ENABLED)
     this._setBoolean(env, 'traceId128BitGenerationEnabled', DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED)
     this._setBoolean(env, 'traceId128BitLoggingEnabled', DD_TRACE_128_BIT_TRACEID_LOGGING_ENABLED)
@@ -786,8 +794,10 @@ class Config {
     this._setBoolean(opts, 'spanRemoveIntegrationFromService', options.spanRemoveIntegrationFromService)
     this._setBoolean(opts, 'startupLogs', options.startupLogs)
     this._setTags(opts, 'tags', tags)
-    this._setBoolean(opts, 'telemetry.logCollection', options.iastOptions &&
-    (options.iastOptions === true || options.iastOptions.enabled === true))
+    const hasTelemetryLogsUsingFeatures =
+      (options.iastOptions && (options.iastOptions === true || options.iastOptions?.enabled === true)) ||
+      (options.profiling && options.profiling === true)
+    this._setBoolean(opts, 'telemetry.logCollection', hasTelemetryLogsUsingFeatures)
     this._setBoolean(opts, 'traceId128BitGenerationEnabled', options.traceId128BitGenerationEnabled)
     this._setBoolean(opts, 'traceId128BitLoggingEnabled', options.traceId128BitLoggingEnabled)
     this._setString(opts, 'version', options.version || tags.version)
@@ -867,7 +877,7 @@ class Config {
     return coalesce(
       this.options.stats,
       process.env.DD_TRACE_STATS_COMPUTATION_ENABLED,
-      getIsGCPFunction() || getIsAzureFunctionConsumptionPlan()
+      getIsGCPFunction() || getIsAzureFunction()
     )
   }
 

package/packages/dd-trace/src/encode/0.4.js

@@ -83,13 +83,17 @@ class AgentEncoder {
       span = formatSpan(span)
       bytes.reserve(1)
 
-      if (span.type) {
+      if (span.type && span.meta_struct) {
+        bytes.buffer[bytes.length++] = 0x8d
+      } else if (span.type || span.meta_struct) {
         bytes.buffer[bytes.length++] = 0x8c
+      } else {
+        bytes.buffer[bytes.length++] = 0x8b
+      }
 
+      if (span.type) {
         this._encodeString(bytes, 'type')
         this._encodeString(bytes, span.type)
-      } else {
-        bytes.buffer[bytes.length++] = 0x8b
       }
 
       this._encodeString(bytes, 'trace_id')
@@ -114,6 +118,10 @@ class AgentEncoder {
       this._encodeMap(bytes, span.meta)
       this._encodeString(bytes, 'metrics')
       this._encodeMap(bytes, span.metrics)
+      if (span.meta_struct) {
+        this._encodeString(bytes, 'meta_struct')
+        this._encodeObject(bytes, span.meta_struct)
+      }
     }
   }
 
@@ -263,6 +271,45 @@ class AgentEncoder {
     }
   }
 
+  _encodeObject (bytes, value, circularReferencesDetector = new Set()) {
+    circularReferencesDetector.add(value)
+    if (Array.isArray(value)) {
+      return this._encodeObjectAsArray(bytes, value, circularReferencesDetector)
+    } else if (value !== null && typeof value === 'object') {
+      return this._encodeObjectAsMap(bytes, value, circularReferencesDetector)
+    } else if (typeof value === 'string' || typeof value === 'number') {
+      this._encodeValue(bytes, value)
+    }
+  }
+
+  _encodeObjectAsMap (bytes, value, circularReferencesDetector) {
+    const keys = Object.keys(value)
+    const validKeys = keys.filter(key =>
+      typeof value[key] === 'string' ||
+      typeof value[key] === 'number' ||
+      (value[key] !== null && typeof value[key] === 'object' && !circularReferencesDetector.has(value[key])))
+
+    this._encodeMapPrefix(bytes, validKeys.length)
+
+    for (const key of validKeys) {
+      this._encodeString(bytes, key)
+      this._encodeObject(bytes, value[key], circularReferencesDetector)
+    }
+  }
+
+  _encodeObjectAsArray (bytes, value, circularReferencesDetector) {
+    const validValue = value.filter(item =>
+      typeof item === 'string' ||
+      typeof item === 'number' ||
+      (item !== null && typeof item === 'object' && !circularReferencesDetector.has(item)))
+
+    this._encodeArrayPrefix(bytes, validValue)
+
+    for (const item of validValue) {
+      this._encodeObject(bytes, item, circularReferencesDetector)
+    }
+  }
+
   _cacheString (value) {
     if (!(value in this._stringMap)) {
       this._stringCount++

package/packages/dd-trace/src/profiling/profiler.js

@@ -5,6 +5,7 @@ const { Config } = require('./config')
 const { snapshotKinds } = require('./constants')
 const { threadNamePrefix } = require('./profilers/shared')
 const dc = require('dc-polyfill')
+const telemetryLog = dc.channel('datadog:telemetry:log')
 
 const profileSubmittedChannel = dc.channel('datadog:profiling:profile-submitted')
 
@@ -15,6 +16,19 @@ function maybeSourceMap (sourceMap, SourceMapper, debug) {
   ], debug)
 }
 
+function logError (logger, err) {
+  if (logger) {
+    logger.error(err)
+  }
+  if (telemetryLog.hasSubscribers) {
+    telemetryLog.publish({
+      message: err.message,
+      level: 'ERROR',
+      stack_trace: err.stack
+    })
+  }
+}
+
 class Profiler extends EventEmitter {
   constructor () {
     super()
@@ -28,13 +42,15 @@ class Profiler extends EventEmitter {
 
   start (options) {
     return this._start(options).catch((err) => {
-      if (options.logger) {
-        options.logger.error(err)
-      }
+      logError(options.logger, err)
       return false
     })
   }
 
+  _logError (err) {
+    logError(this._logger, err)
+  }
+
   async _start (options) {
     if (this._enabled) return true
 
@@ -61,7 +77,7 @@ class Profiler extends EventEmitter {
         })
       }
     } catch (err) {
-      this._logger.error(err)
+      this._logError(err)
     }
 
     try {
@@ -78,7 +94,7 @@ class Profiler extends EventEmitter {
       this._capture(this._timeoutInterval, start)
       return true
     } catch (e) {
-      this._logger.error(e)
+      this._logError(e)
       this._stop()
       return false
     }
@@ -167,7 +183,7 @@ class Profiler extends EventEmitter {
       profileSubmittedChannel.publish()
       this._logger.debug('Submitted profiles')
     } catch (err) {
-      this._logger.error(err)
+      this._logError(err)
       this._stop()
     }
   }
@@ -182,7 +198,7 @@ class Profiler extends EventEmitter {
     tags.snapshot = snapshotKind
     for (const exporter of this._config.exporters) {
       const task = exporter.export({ profiles, start, end, tags })
-        .catch(err => this._logger.error(err))
+        .catch(err => this._logError(err))
 
       tasks.push(task)
     }

package/packages/dd-trace/src/proxy.js

@@ -14,6 +14,7 @@ const dogstatsd = require('./dogstatsd')
 const NoopDogStatsDClient = require('./noop/dogstatsd')
 const spanleak = require('./spanleak')
 const { SSITelemetry } = require('./profiling/ssi-telemetry')
+const telemetryLog = require('dc-polyfill').channel('datadog:telemetry:log')
 
 class LazyModule {
   constructor (provider) {
@@ -91,7 +92,7 @@ class Tracer extends NoopProxy {
         })
       }
 
-      if (config.isGCPFunction || config.isAzureFunctionConsumptionPlan) {
+      if (config.isGCPFunction || config.isAzureFunction) {
         require('./serverless').maybeStartServerlessMiniAgent(config)
       }
 
@@ -104,6 +105,11 @@ class Tracer extends NoopProxy {
           this._profilerStarted = profiler.start(config)
         } catch (e) {
           log.error(e)
+          telemetryLog.publish({
+            message: e.message,
+            level: 'ERROR',
+            stack_trace: e.stack
+          })
         }
       } else if (ssiTelemetry.enabled()) {
         require('./profiling/ssi-telemetry-mock-profiler').start(config)

package/packages/dd-trace/src/serverless.js

@@ -53,18 +53,16 @@ function getIsGCPFunction () {
   return isDeprecatedGCPFunction || isNewerGCPFunction
 }
 
-function getIsAzureFunctionConsumptionPlan () {
+function getIsAzureFunction () {
   const isAzureFunction =
     process.env.FUNCTIONS_EXTENSION_VERSION !== undefined && process.env.FUNCTIONS_WORKER_RUNTIME !== undefined
-  const azureWebsiteSKU = process.env.WEBSITE_SKU
-  const isConsumptionPlan = azureWebsiteSKU === undefined || azureWebsiteSKU === 'Dynamic'
 
-  return isAzureFunction && isConsumptionPlan
+  return isAzureFunction
 }
 
 module.exports = {
   maybeStartServerlessMiniAgent,
   getIsGCPFunction,
-  getIsAzureFunctionConsumptionPlan,
+  getIsAzureFunction,
   getRustBinaryPath
 }

package/packages/dd-trace/src/telemetry/logs/log-collector.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const log = require('../../log')
+const { calculateDDBasePath } = require('../../util')
 
 const logs = new Map()
 
@@ -29,6 +30,37 @@ function isValid (logEntry) {
   return logEntry?.level && logEntry.message
 }
 
+const ddBasePath = calculateDDBasePath(__dirname)
+const EOL = '\n'
+const STACK_FRAME_LINE_REGEX = /^\s*at\s/gm
+
+function sanitize (logEntry) {
+  const stack = logEntry.stack_trace
+  if (!stack) return logEntry
+
+  let stackLines = stack.split(EOL)
+
+  const firstIndex = stackLines.findIndex(l => l.match(STACK_FRAME_LINE_REGEX))
+
+  const isDDCode = firstIndex > -1 && stackLines[firstIndex].includes(ddBasePath)
+  stackLines = stackLines
+    .filter((line, index) => (isDDCode && index < firstIndex) || line.includes(ddBasePath))
+    .map(line => line.replace(ddBasePath, ''))
+
+  logEntry.stack_trace = stackLines.join(EOL)
+  if (logEntry.stack_trace === '') {
+    // If entire stack was removed, we'd just have a message saying "omitted"
+    // in which case we'd rather not log it at all.
+    return null
+  }
+
+  if (!isDDCode) {
+    logEntry.message = 'omitted'
+  }
+
+  return logEntry
+}
+
 const logCollector = {
   add (logEntry) {
     try {
@@ -37,9 +69,13 @@ const logCollector = {
       // NOTE: should errors have higher priority? and discard log entries with lower priority?
       if (logs.size >= maxEntries) {
         overflowedCount++
-        return
+        return false
       }
 
+      logEntry = sanitize(logEntry)
+      if (!logEntry) {
+        return false
+      }
       const hash = createHash(logEntry)
       if (!logs.has(hash)) {
         logs.set(hash, logEntry)
@@ -51,6 +87,11 @@ const logCollector = {
     return false
   },
 
+  // Used for testing
+  hasEntry (logEntry) {
+    return logs.has(createHash(logEntry))
+  },
+
   drain () {
     if (logs.size === 0) return