dd-trace 4.46.0 → 4.47.0

package/packages/dd-trace/src/config.js

@@ -3,7 +3,7 @@
 const fs = require('fs')
 const os = require('os')
 const uuid = require('crypto-randomuuid') // we need to keep the old uuid dep because of cypress
-const URL = require('url').URL
+const { URL } = require('url')
 const log = require('./log')
 const pkg = require('./pkg')
 const coalesce = require('koalas')
@@ -18,6 +18,7 @@ const { updateConfig } = require('./telemetry')
 const telemetryMetrics = require('./telemetry/metrics')
 const { getIsGCPFunction, getIsAzureFunction } = require('./serverless')
 const { ORIGIN_KEY } = require('./constants')
+const { appendRules } = require('./payload-tagging/config')
 
 const tracerMetrics = telemetryMetrics.manager.namespace('tracers')
 
@@ -173,6 +174,21 @@ function validateNamingVersion (versionString) {
   return versionString
 }
 
+/**
+ * Given a string of comma-separated paths, return the array of paths.
+ * If a blank path is provided a null is returned to signal that the feature is disabled.
+ * An empty array means the feature is enabled but that no rules need to be applied.
+ *
+ * @param {string} input
+ * @returns {[string]|null}
+ */
+function splitJSONPathRules (input) {
+  if (!input) return null
+  if (Array.isArray(input)) return input
+  if (input === 'all') return []
+  return input.split(',')
+}
+
 // Shallow clone with property name remapping
 function remapify (input, mappings) {
   if (!input) return
@@ -281,6 +297,26 @@ class Config {
       null
     )
 
+    const DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING = splitJSONPathRules(
+      coalesce(
+        process.env.DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING,
+        options.cloudPayloadTagging?.request,
+        ''
+      ))
+
+    const DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING = splitJSONPathRules(
+      coalesce(
+        process.env.DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING,
+        options.cloudPayloadTagging?.response,
+        ''
+      ))
+
+    const DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH = coalesce(
+      process.env.DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH,
+      options.cloudPayloadTagging?.maxDepth,
+      10
+    )
+
     // TODO: refactor
     this.apiKey = DD_API_KEY
 
@@ -291,6 +327,15 @@ class Config {
       type: DD_INSTRUMENTATION_INSTALL_TYPE
     }
 
+    this.cloudPayloadTagging = {
+      requestsEnabled: !!DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING,
+      responsesEnabled: !!DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING,
+      maxDepth: DD_TRACE_CLOUD_PAYLOAD_TAGGING_MAX_DEPTH,
+      rules: appendRules(
+        DD_TRACE_CLOUD_REQUEST_PAYLOAD_TAGGING, DD_TRACE_CLOUD_RESPONSE_PAYLOAD_TAGGING
+      )
+    }
+
     this._applyDefaults()
     this._applyEnvironment()
     this._applyOptions(options)
@@ -423,6 +468,7 @@ class Config {
     this._setValue(defaults, 'dogstatsd.hostname', '127.0.0.1')
     this._setValue(defaults, 'dogstatsd.port', '8125')
     this._setValue(defaults, 'dsmEnabled', false)
+    this._setValue(defaults, 'dynamicInstrumentationEnabled', false)
     this._setValue(defaults, 'env', undefined)
     this._setValue(defaults, 'experimental.enableGetRumData', false)
     this._setValue(defaults, 'experimental.exporter', undefined)
@@ -451,6 +497,7 @@ class Config {
     this._setValue(defaults, 'isGitUploadEnabled', false)
     this._setValue(defaults, 'isIntelligentTestRunnerEnabled', false)
     this._setValue(defaults, 'isManualApiEnabled', false)
+    this._setValue(defaults, 'ciVisibilityTestSessionName', '')
     this._setValue(defaults, 'logInjection', false)
     this._setValue(defaults, 'lookup', undefined)
     this._setValue(defaults, 'memcachedCommandEnabled', false)
@@ -528,6 +575,7 @@ class Config {
       DD_DBM_PROPAGATION_MODE,
       DD_DOGSTATSD_HOSTNAME,
       DD_DOGSTATSD_PORT,
+      DD_DYNAMIC_INSTRUMENTATION_ENABLED,
       DD_ENV,
       DD_EXPERIMENTAL_API_SECURITY_ENABLED,
       DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED,
@@ -657,6 +705,7 @@ class Config {
     this._setString(env, 'dogstatsd.hostname', DD_DOGSTATSD_HOSTNAME)
     this._setString(env, 'dogstatsd.port', DD_DOGSTATSD_PORT)
     this._setBoolean(env, 'dsmEnabled', DD_DATA_STREAMS_ENABLED)
+    this._setBoolean(env, 'dynamicInstrumentationEnabled', DD_DYNAMIC_INSTRUMENTATION_ENABLED)
     this._setString(env, 'env', DD_ENV || tags.env)
     this._setBoolean(env, 'experimental.enableGetRumData', DD_TRACE_EXPERIMENTAL_GET_RUM_DATA_ENABLED)
     this._setString(env, 'experimental.exporter', DD_TRACE_EXPERIMENTAL_EXPORTER)
@@ -824,11 +873,11 @@ class Config {
       this._setString(opts, 'dogstatsd.port', options.dogstatsd.port)
     }
     this._setBoolean(opts, 'dsmEnabled', options.dsmEnabled)
+    this._setBoolean(opts, 'dynamicInstrumentationEnabled', options.experimental?.dynamicInstrumentationEnabled)
     this._setString(opts, 'env', options.env || tags.env)
-    this._setBoolean(opts, 'experimental.enableGetRumData',
-      options.experimental && options.experimental.enableGetRumData)
-    this._setString(opts, 'experimental.exporter', options.experimental && options.experimental.exporter)
-    this._setBoolean(opts, 'experimental.runtimeId', options.experimental && options.experimental.runtimeId)
+    this._setBoolean(opts, 'experimental.enableGetRumData', options.experimental?.enableGetRumData)
+    this._setString(opts, 'experimental.exporter', options.experimental?.exporter)
+    this._setBoolean(opts, 'experimental.runtimeId', options.experimental?.runtimeId)
     this._setValue(opts, 'flushInterval', maybeInt(options.flushInterval))
     this._optsUnprocessed.flushInterval = options.flushInterval
     this._setValue(opts, 'flushMinSpans', maybeInt(options.flushMinSpans))
@@ -954,10 +1003,10 @@ class Config {
   }
 
   _isCiVisibilityManualApiEnabled () {
-    return isTrue(coalesce(
+    return coalesce(
       process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED,
-      false
-    ))
+      true
+    )
   }
 
   _isTraceStatsComputationEnabled () {
@@ -985,7 +1034,8 @@ class Config {
       DD_CIVISIBILITY_AGENTLESS_URL,
       DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED,
       DD_CIVISIBILITY_FLAKY_RETRY_ENABLED,
-      DD_CIVISIBILITY_FLAKY_RETRY_COUNT
+      DD_CIVISIBILITY_FLAKY_RETRY_COUNT,
+      DD_TEST_SESSION_NAME
     } = process.env
 
     if (DD_CIVISIBILITY_AGENTLESS_URL) {
@@ -1000,7 +1050,8 @@ class Config {
         coalesce(DD_CIVISIBILITY_FLAKY_RETRY_ENABLED, true))
       this._setValue(calc, 'flakyTestRetriesCount', coalesce(maybeInt(DD_CIVISIBILITY_FLAKY_RETRY_COUNT), 5))
       this._setBoolean(calc, 'isIntelligentTestRunnerEnabled', isTrue(this._isCiVisibilityItrEnabled()))
-      this._setBoolean(calc, 'isManualApiEnabled', this._isCiVisibilityManualApiEnabled())
+      this._setBoolean(calc, 'isManualApiEnabled', !isFalse(this._isCiVisibilityManualApiEnabled()))
+      this._setString(calc, 'ciVisibilityTestSessionName', DD_TEST_SESSION_NAME)
     }
     this._setString(calc, 'dogstatsd.hostname', this._getHostname())
     this._setBoolean(calc, 'isGitUploadEnabled',

package/packages/dd-trace/src/constants.js

@@ -34,5 +34,15 @@ module.exports = {
   SCI_REPOSITORY_URL: '_dd.git.repository_url',
   SCI_COMMIT_SHA: '_dd.git.commit.sha',
   APM_TRACING_ENABLED_KEY: '_dd.apm.enabled',
-  APPSEC_PROPAGATION_KEY: '_dd.p.appsec'
+  APPSEC_PROPAGATION_KEY: '_dd.p.appsec',
+  PAYLOAD_TAG_REQUEST_PREFIX: 'aws.request.body',
+  PAYLOAD_TAG_RESPONSE_PREFIX: 'aws.response.body',
+  PAYLOAD_TAGGING_MAX_TAGS: 758,
+  SCHEMA_DEFINITION: 'schema.definition',
+  SCHEMA_WEIGHT: 'schema.weight',
+  SCHEMA_TYPE: 'schema.type',
+  SCHEMA_ID: 'schema.id',
+  SCHEMA_TOPIC: 'schema.topic',
+  SCHEMA_OPERATION: 'schema.operation',
+  SCHEMA_NAME: 'schema.name'
 }

package/packages/dd-trace/src/data_streams_context.js

@@ -1,4 +1,5 @@
 const { storage } = require('../../datadog-core')
+const log = require('./log')
 
 function getDataStreamsContext () {
   const store = storage.getStore()
@@ -6,6 +7,8 @@ function getDataStreamsContext () {
 }
 
 function setDataStreamsContext (dataStreamsContext) {
+  log.debug(() => `Setting new DSM Context: ${JSON.stringify(dataStreamsContext)}.`)
+
   if (dataStreamsContext) storage.enterWith({ ...(storage.getStore()), dataStreamsContext })
 }
 

package/packages/dd-trace/src/datastreams/fnv.js

@@ -0,0 +1,23 @@
+const FNV_64_PRIME = BigInt('0x100000001B3')
+const FNV1_64_INIT = BigInt('0xCBF29CE484222325')
+
+function fnv (data, hvalInit, fnvPrime, fnvSize) {
+  let hval = hvalInit
+  for (const byte of data) {
+    hval = (hval * fnvPrime) % fnvSize
+    hval = hval ^ BigInt(byte)
+  }
+  return hval
+}
+
+function fnv64 (data) {
+  if (!Buffer.isBuffer(data)) {
+    data = Buffer.from(data, 'utf-8')
+  }
+  const byteArray = new Uint8Array(data)
+  return fnv(byteArray, FNV1_64_INIT, FNV_64_PRIME, BigInt(2) ** BigInt(64))
+}
+
+module.exports = {
+  fnv64
+}

package/packages/dd-trace/src/datastreams/pathway.js

@@ -4,6 +4,8 @@
 const crypto = require('crypto')
 const { encodeVarint, decodeVarint } = require('./encoding')
 const LRUCache = require('lru-cache')
+const log = require('../log')
+const pick = require('../../../datadog-core/src/utils/src/pick')
 
 const options = { max: 500 }
 const cache = new LRUCache(options)
@@ -11,6 +13,8 @@ const cache = new LRUCache(options)
 const CONTEXT_PROPAGATION_KEY = 'dd-pathway-ctx'
 const CONTEXT_PROPAGATION_KEY_BASE64 = 'dd-pathway-ctx-base64'
 
+const logKeys = [CONTEXT_PROPAGATION_KEY, CONTEXT_PROPAGATION_KEY_BASE64]
+
 function shaHash (checkpointString) {
   const hash = crypto.createHash('md5').update(checkpointString).digest('hex').slice(0, 16)
   return Buffer.from(hash, 'hex')
@@ -80,9 +84,13 @@ class DsmPathwayCodec {
       return
     }
     carrier[CONTEXT_PROPAGATION_KEY_BASE64] = encodePathwayContextBase64(dataStreamsContext)
+
+    log.debug(() => `Injected into DSM carrier: ${JSON.stringify(pick(carrier, logKeys))}.`)
   }
 
   static decode (carrier) {
+    log.debug(() => `Attempting extract from DSM carrier: ${JSON.stringify(pick(carrier, logKeys))}.`)
+
     if (carrier == null) return
 
     let ctx
@@ -97,13 +105,12 @@ class DsmPathwayCodec {
         // pass
       }
       // cover case where base64 context was received under wrong key
-      if (!ctx) ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY])
+      if (!ctx && CONTEXT_PROPAGATION_KEY in carrier) {
+        ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY])
+      }
     }
-    return ctx
-  }
 
-  static contextExists (carrier) {
-    return CONTEXT_PROPAGATION_KEY_BASE64 in carrier || CONTEXT_PROPAGATION_KEY in carrier
+    return ctx
   }
 }
 

package/packages/dd-trace/src/datastreams/processor.js

@@ -9,6 +9,9 @@ const { DataStreamsWriter } = require('./writer')
 const { computePathwayHash } = require('./pathway')
 const { types } = require('util')
 const { PATHWAY_HASH } = require('../../../../ext/tags')
+const { SchemaBuilder } = require('./schemas/schema_builder')
+const { SchemaSampler } = require('./schemas/schema_sampler')
+const log = require('../log')
 
 const ENTRY_PARENT_HASH = Buffer.from('0000000000000000', 'hex')
 
@@ -194,6 +197,7 @@ class DataStreamsProcessor {
     this.version = version || ''
     this.sequence = 0
     this.flushInterval = flushInterval
+    this._schemaSamplers = {}
 
     if (this.enabled) {
       this.timer = setInterval(this.onInterval.bind(this), flushInterval)
@@ -269,6 +273,11 @@ class DataStreamsProcessor {
         closestOppositeDirectionHash = parentHash
         closestOppositeDirectionEdgeStart = edgeStartNs
       }
+      log.debug(
+        () => `Setting DSM Checkpoint from extracted parent context with hash: ${parentHash} and edge tags: ${edgeTags}`
+      )
+    } else {
+      log.debug(() => 'Setting DSM Checkpoint with empty parent context.')
     }
     const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash)
     const edgeLatencyNs = nowNs - edgeStartNs
@@ -352,6 +361,32 @@ class DataStreamsProcessor {
   setUrl (url) {
     this.writer.setUrl(url)
   }
+
+  trySampleSchema (topic) {
+    const nowMs = Date.now()
+
+    if (!this._schemaSamplers[topic]) {
+      this._schemaSamplers[topic] = new SchemaSampler()
+    }
+
+    const sampler = this._schemaSamplers[topic]
+    return sampler.trySample(nowMs)
+  }
+
+  canSampleSchema (topic) {
+    const nowMs = Date.now()
+
+    if (!this._schemaSamplers[topic]) {
+      this._schemaSamplers[topic] = new SchemaSampler()
+    }
+
+    const sampler = this._schemaSamplers[topic]
+    return sampler.canSample(nowMs)
+  }
+
+  getSchema (schemaName, iterator) {
+    return SchemaBuilder.getSchema(schemaName, iterator)
+  }
 }
 
 module.exports = {

package/packages/dd-trace/src/datastreams/schemas/schema.js

@@ -0,0 +1,8 @@
+class Schema {
+  constructor (definition, id) {
+    this.definition = definition
+    this.id = id
+  }
+}
+
+module.exports = { Schema }

package/packages/dd-trace/src/datastreams/schemas/schema_builder.js

@@ -0,0 +1,125 @@
+const LRUCache = require('lru-cache')
+const { fnv64 } = require('../fnv')
+const { Schema } = require('./schema')
+
+const maxDepth = 10
+const maxProperties = 1000
+const CACHE = new LRUCache({ max: 32 })
+
+class SchemaBuilder {
+  constructor (iterator) {
+    this.schema = new OpenApiSchema()
+    this.iterator = iterator
+    this.proerties = 0
+  }
+
+  addProperty (schemaName, fieldName, isArray, type, description, ref, format, enumValues) {
+    if (this.properties >= maxProperties) {
+      return false
+    }
+    this.properties += 1
+    let property = new OpenApiSchema.PROPERTY(type, description, ref, format, enumValues, null)
+    if (isArray) {
+      property = new OpenApiSchema.PROPERTY('array', null, null, null, null, property)
+    }
+    this.schema.components.schemas[schemaName].properties[fieldName] = property
+    return true
+  }
+
+  build () {
+    this.iterator.iterateOverSchema(this)
+    const noNones = convertToJsonCompatible(this.schema)
+    const definition = jsonStringify(noNones)
+    const id = fnv64(Buffer.from(definition, 'utf-8')).toString()
+    return new Schema(definition, id)
+  }
+
+  shouldExtractSchema (schemaName, depth) {
+    if (depth > maxDepth) {
+      return false
+    }
+    if (schemaName in this.schema.components.schemas) {
+      return false
+    }
+    this.schema.components.schemas[schemaName] = new OpenApiSchema.SCHEMA()
+    return true
+  }
+
+  static getSchema (schemaName, iterator) {
+    if (!CACHE.has(schemaName)) {
+      CACHE.set(schemaName, new SchemaBuilder(iterator).build())
+    }
+    return CACHE.get(schemaName)
+  }
+}
+
+class OpenApiSchema {
+  constructor () {
+    this.openapi = '3.0.0'
+    this.components = new OpenApiComponents()
+  }
+}
+
+OpenApiSchema.SCHEMA = class {
+  constructor () {
+    this.type = 'object'
+    this.properties = {}
+  }
+}
+
+OpenApiSchema.PROPERTY = class {
+  constructor (type, description = null, ref = null, format = null, enumValues = null, items = null) {
+    this.type = type
+    this.description = description
+    this.$ref = ref
+    this.format = format
+    this.enum = enumValues
+    this.items = items
+  }
+}
+
+class OpenApiComponents {
+  constructor () {
+    this.schemas = {}
+  }
+}
+
+function convertToJsonCompatible (obj) {
+  if (Array.isArray(obj)) {
+    return obj.filter(item => item !== null).map(item => convertToJsonCompatible(item))
+  } else if (obj && typeof obj === 'object') {
+    const jsonObj = {}
+    for (const [key, value] of Object.entries(obj)) {
+      if (value !== null) {
+        jsonObj[key] = convertToJsonCompatible(value)
+      }
+    }
+    return jsonObj
+  }
+  return obj
+}
+
+function convertKey (key) {
+  if (key === 'enumValues') {
+    return 'enum'
+  }
+  return key
+}
+
+function jsonStringify (obj, indent = 2) {
+  // made to stringify json exactly similar to python / java in order for hashing to be the same
+  const jsonString = JSON.stringify(obj, (_, value) => value, indent)
+  return jsonString.replace(/^ +/gm, ' ') // Replace leading spaces with single space
+    .replace(/\n/g, '') // Remove newlines
+    .replace(/{ /g, '{') // Remove space after '{'
+    .replace(/ }/g, '}') // Remove space before '}'
+    .replace(/\[ /g, '[') // Remove space after '['
+    .replace(/ \]/g, ']') // Remove space before ']'
+}
+
+module.exports = {
+  SchemaBuilder,
+  OpenApiSchema,
+  convertToJsonCompatible,
+  convertKey
+}

package/packages/dd-trace/src/datastreams/schemas/schema_sampler.js

@@ -0,0 +1,29 @@
+const SAMPLE_INTERVAL_MILLIS = 30 * 1000
+
+class SchemaSampler {
+  constructor () {
+    this.weight = 0
+    this.lastSampleMs = 0
+  }
+
+  trySample (currentTimeMs) {
+    if (currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS) {
+      if (currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS) {
+        this.lastSampleMs = currentTimeMs
+        const weight = this.weight
+        this.weight = 0
+        return weight
+      }
+    }
+    return 0
+  }
+
+  canSample (currentTimeMs) {
+    this.weight += 1
+    return currentTimeMs >= this.lastSampleMs + SAMPLE_INTERVAL_MILLIS
+  }
+}
+
+module.exports = {
+  SchemaSampler
+}

package/packages/dd-trace/src/debugger/devtools_client/config.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const { workerData: { config: parentConfig, parentThreadId, configPort } } = require('node:worker_threads')
+const { format } = require('node:url')
+const log = require('../../log')
+
+const config = module.exports = {
+  runtimeId: parentConfig.tags['runtime-id'],
+  service: parentConfig.service,
+  parentThreadId
+}
+
+updateUrl(parentConfig)
+
+configPort.on('message', updateUrl)
+configPort.on('messageerror', (err) => log.error(err))
+
+function updateUrl (updates) {
+  config.url = updates.url || format({
+    protocol: 'http:',
+    hostname: updates.hostname || 'localhost',
+    port: updates.port
+  })
+}

package/packages/dd-trace/src/debugger/devtools_client/index.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const { randomUUID } = require('crypto')
+const { breakpoints } = require('./state')
+const session = require('./session')
+const send = require('./send')
+const { ackEmitting } = require('./status')
+const { parentThreadId } = require('./config')
+const log = require('../../log')
+const { version } = require('../../../../../package.json')
+
+require('./remote_config')
+
+// There doesn't seem to be an official standard for the content of these fields, so we're just populating them with
+// something that should be useful to a Node.js developer.
+const threadId = parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${parentThreadId}`
+const threadName = parentThreadId === 0 ? 'MainThread' : `WorkerThread:${parentThreadId}`
+
+session.on('Debugger.paused', async ({ params }) => {
+  const start = process.hrtime.bigint()
+  const timestamp = Date.now()
+  const probes = params.hitBreakpoints.map((id) => breakpoints.get(id))
+  await session.post('Debugger.resume')
+  const diff = process.hrtime.bigint() - start // TODO: Should this be recored as telemetry?
+
+  log.debug(`Finished processing breakpoints - main thread paused for: ${Number(diff) / 1000000} ms`)
+
+  const logger = {
+    // We can safely use `location.file` from the first probe in the array, since all probes hit by `hitBreakpoints`
+    // must exist in the same file since the debugger can only pause the main thread in one location.
+    name: probes[0].location.file, // name of the class/type/file emitting the snapshot
+    method: params.callFrames[0].functionName, // name of the method/function emitting the snapshot
+    version,
+    thread_id: threadId,
+    thread_name: threadName
+  }
+
+  // TODO: Send multiple probes in one HTTP request as an array
+  for (const probe of probes) {
+    const snapshot = {
+      id: randomUUID(),
+      timestamp,
+      probe: {
+        id: probe.id,
+        version: probe.version,
+        location: probe.location
+      },
+      language: 'javascript'
+    }
+
+    // TODO: Process template
+    send(probe.template, logger, snapshot, (err) => {
+      if (err) log.error(err)
+      else ackEmitting(probe)
+    })
+  }
+})

package/packages/dd-trace/src/debugger/devtools_client/inspector_promises_polyfill.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const { builtinModules } = require('node:module')
+
+if (builtinModules.includes('inspector/promises')) {
+  module.exports = require('node:inspector/promises')
+} else {
+  const inspector = require('node:inspector')
+  const { promisify } = require('node:util')
+
+  // The rest of the code in this file is lifted from:
+  // https://github.com/nodejs/node/blob/1d4d76ff3fb08f9a0c55a1d5530b46c4d5d550c7/lib/inspector/promises.js
+  class Session extends inspector.Session {
+    constructor () { super() } // eslint-disable-line no-useless-constructor
+  }
+
+  Session.prototype.post = promisify(inspector.Session.prototype.post)
+
+  module.exports = {
+    ...inspector,
+    Session
+  }
+}