dd-trace 3.24.0 → 3.26.0

package/packages/dd-trace/src/datastreams/processor.js

@@ -0,0 +1,194 @@
+const os = require('os')
+const pkg = require('../../../../package.json')
+// Message pack int encoding is done in big endian, but data streams uses little endian
+const Uint64 = require('int64-buffer').Uint64BE
+
+const { LogCollapsingLowestDenseDDSketch } = require('@datadog/sketches-js')
+
+const { DataStreamsWriter } = require('./writer')
+const { computePathwayHash } = require('./pathway')
+const ENTRY_PARENT_HASH = Buffer.from('0000000000000000', 'hex')
+
+const HIGH_ACCURACY_DISTRIBUTION = 0.0075
+
+class StatsPoint {
+  constructor (hash, parentHash, edgeTags) {
+    this.hash = new Uint64(hash)
+    this.parentHash = new Uint64(parentHash)
+    this.edgeTags = edgeTags
+    this.edgeLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
+    this.pathwayLatency = new LogCollapsingLowestDenseDDSketch(HIGH_ACCURACY_DISTRIBUTION)
+  }
+
+  addLatencies (checkpoint) {
+    const edgeLatencySec = checkpoint.edgeLatencyNs / 1e9
+    const pathwayLatencySec = checkpoint.pathwayLatencyNs / 1e9
+    this.edgeLatency.accept(edgeLatencySec)
+    this.pathwayLatency.accept(pathwayLatencySec)
+  }
+
+  encode () {
+    return {
+      Hash: this.hash,
+      ParentHash: this.parentHash,
+      EdgeTags: this.edgeTags,
+      EdgeLatency: this.edgeLatency.toProto(),
+      PathwayLatency: this.pathwayLatency.toProto()
+    }
+  }
+}
+
+class StatsBucket extends Map {
+  forCheckpoint (checkpoint) {
+    const key = checkpoint.hash
+    if (!this.has(key)) {
+      this.set(key, new StatsPoint(checkpoint.hash, checkpoint.parentHash, checkpoint.edgeTags)) // StatsPoint
+    }
+
+    return this.get(key)
+  }
+}
+
+class TimeBuckets extends Map {
+  forTime (time) {
+    if (!this.has(time)) {
+      this.set(time, new StatsBucket())
+    }
+
+    return this.get(time)
+  }
+}
+
+class DataStreamsProcessor {
+  constructor ({
+    dsmEnabled,
+    hostname,
+    port,
+    url,
+    env,
+    tags
+  } = {}) {
+    this.writer = new DataStreamsWriter({
+      hostname,
+      port,
+      url
+    })
+    this.bucketSizeNs = 1e10
+    this.buckets = new TimeBuckets()
+    this.hostname = os.hostname()
+    this.enabled = dsmEnabled
+    this.env = env
+    this.tags = tags || {}
+    this.service = this.tags.service || 'unnamed-nodejs-service'
+    this.sequence = 0
+
+    if (this.enabled) {
+      this.timer = setInterval(this.onInterval.bind(this), 10000)
+      this.timer.unref()
+    }
+  }
+
+  onInterval () {
+    const serialized = this._serializeBuckets()
+    if (!serialized) return
+    const payload = {
+      Env: this.env,
+      Service: this.service,
+      Stats: serialized,
+      TracerVersion: pkg.version,
+      Lang: 'javascript'
+    }
+    this.writer.flush(payload)
+  }
+
+  recordCheckpoint (checkpoint) {
+    if (!this.enabled) return
+    const bucketTime = Math.round(checkpoint.currentTimestamp - (checkpoint.currentTimestamp % this.bucketSizeNs))
+    this.buckets.forTime(bucketTime)
+      .forCheckpoint(checkpoint)
+      .addLatencies(checkpoint)
+  }
+
+  setCheckpoint (edgeTags, ctx = null) {
+    if (!this.enabled) return null
+    const nowNs = Date.now() * 1e6
+    const direction = edgeTags.find(t => t.startsWith('direction:'))
+    let pathwayStartNs = nowNs
+    let edgeStartNs = nowNs
+    let parentHash = ENTRY_PARENT_HASH
+    let closestOppositeDirectionHash = ENTRY_PARENT_HASH
+    let closestOppositeDirectionEdgeStart = nowNs
+    if (ctx != null) {
+      pathwayStartNs = ctx.pathwayStartNs
+      edgeStartNs = ctx.edgeStartNs
+      parentHash = ctx.hash
+      closestOppositeDirectionHash = ctx.closestOppositeDirectionHash || ENTRY_PARENT_HASH
+      closestOppositeDirectionEdgeStart = ctx.closestOppositeDirectionEdgeStart || nowNs
+      if (direction === ctx.previousDirection) {
+        parentHash = ctx.closestOppositeDirectionHash
+        if (parentHash === ENTRY_PARENT_HASH) {
+          // if the closest hash from opposite direction is the entry hash, that means
+          // we produce in a loop, without consuming
+          // in that case, we don't want the pathway to be longer and longer, but we want to restart a new pathway.
+          edgeStartNs = nowNs
+          pathwayStartNs = nowNs
+        } else {
+          edgeStartNs = ctx.closestOppositeDirectionEdgeStart
+        }
+      } else {
+        closestOppositeDirectionHash = parentHash
+        closestOppositeDirectionEdgeStart = edgeStartNs
+      }
+    }
+    const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash)
+    const edgeLatencyNs = nowNs - edgeStartNs
+    const pathwayLatencyNs = nowNs - pathwayStartNs
+    const checkpoint = {
+      currentTimestamp: nowNs,
+      parentHash: parentHash,
+      hash: hash,
+      edgeTags: edgeTags,
+      edgeLatencyNs: edgeLatencyNs,
+      pathwayLatencyNs: pathwayLatencyNs
+    }
+    this.recordCheckpoint(checkpoint)
+    return {
+      hash: hash,
+      edgeStartNs: edgeStartNs,
+      pathwayStartNs: pathwayStartNs,
+      previousDirection: direction,
+      closestOppositeDirectionHash: closestOppositeDirectionHash,
+      closestOppositeDirectionEdgeStart: closestOppositeDirectionEdgeStart
+    }
+  }
+
+  _serializeBuckets () {
+    const serializedBuckets = []
+
+    for (const [ timeNs, bucket ] of this.buckets.entries()) {
+      const points = []
+
+      for (const stats of bucket.values()) {
+        points.push(stats.encode())
+      }
+
+      serializedBuckets.push({
+        Start: new Uint64(timeNs),
+        Duration: new Uint64(this.bucketSizeNs),
+        Stats: points
+      })
+    }
+
+    this.buckets.clear()
+
+    return serializedBuckets
+  }
+}
+
+module.exports = {
+  DataStreamsProcessor: DataStreamsProcessor,
+  StatsPoint: StatsPoint,
+  StatsBucket: StatsBucket,
+  TimeBuckets,
+  ENTRY_PARENT_HASH
+}

package/packages/dd-trace/src/datastreams/writer.js

@@ -0,0 +1,66 @@
+const pkg = require('../../../../package.json')
+const log = require('../log')
+const request = require('../exporters/common/request')
+const { URL, format } = require('url')
+const msgpack = require('msgpack-lite')
+const zlib = require('zlib')
+const codec = msgpack.createCodec({ int64: true })
+
+function makeRequest (data, url, cb) {
+  const options = {
+    path: '/v0.1/pipeline_stats',
+    method: 'POST',
+    headers: {
+      'Datadog-Meta-Lang': 'javascript',
+      'Datadog-Meta-Tracer-Version': pkg.version,
+      'Content-Type': 'application/msgpack',
+      'Content-Encoding': 'gzip'
+    }
+  }
+
+  options.protocol = url.protocol
+  options.hostname = url.hostname
+  options.port = url.port
+
+  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
+
+  request(data, options, (err, res) => {
+    cb(err, res)
+  })
+}
+
+class DataStreamsWriter {
+  constructor (config) {
+    const { hostname = '127.0.0.1', port = 8126, url } = config
+    this._url = url || new URL(format({
+      protocol: 'http:',
+      hostname: hostname || 'localhost',
+      port
+    }))
+  }
+
+  flush (payload) {
+    if (!request.writable) {
+      log.debug(() => `Maximum number of active requests reached. Payload discarded: ${JSON.stringify(payload)}`)
+      return
+    }
+    const encodedPayload = msgpack.encode(payload, { codec })
+
+    zlib.gzip(encodedPayload, { level: 1 }, (err, compressedData) => {
+      if (err) {
+        log.error(err)
+        return
+      }
+      makeRequest(compressedData, this._url, (err, res) => {
+        log.debug(`Response from the agent: ${res}`)
+        if (err) {
+          log.error(err)
+        }
+      })
+    })
+  }
+}
+
+module.exports = {
+  DataStreamsWriter
+}

package/packages/dd-trace/src/dogstatsd.js

@@ -8,7 +8,11 @@ const log = require('./log')
 
 const MAX_BUFFER_SIZE = 1024 // limit from the agent
 
-class Client {
+const TYPE_COUNTER = 'c'
+const TYPE_GAUGE = 'g'
+const TYPE_DISTRIBUTION = 'd'
+
+class DogStatsDClient {
   constructor (options) {
     options = options || {}
 
@@ -32,11 +36,15 @@ class Client {
   }
 
   gauge (stat, value, tags) {
-    this._add(stat, value, 'g', tags)
+    this._add(stat, value, TYPE_GAUGE, tags)
   }
 
   increment (stat, value, tags) {
-    this._add(stat, value, 'c', tags)
+    this._add(stat, value, TYPE_COUNTER, tags)
+  }
+
+  distribution (stat, value, tags) {
+    this._add(stat, value, TYPE_DISTRIBUTION, tags)
   }
 
   flush () {
@@ -135,4 +143,4 @@ class Client {
   }
 }
 
-module.exports = Client
+module.exports = DogStatsDClient

package/packages/dd-trace/src/external-logger/src/index.js

@@ -7,6 +7,8 @@ class ExternalLogger {
   constructor ({
     ddsource, hostname, service, apiKey, site = 'datadoghq.com', interval = 10000, timeout = 2000, limit = 1000
   }) {
+    this.enabled = !!apiKey
+
     this.ddsource = ddsource
     this.hostname = hostname
     this.service = service
@@ -38,6 +40,8 @@ class ExternalLogger {
 
   // Parses and enqueues a log
   log (log, span, tags) {
+    if (!this.enabled) return
+
     const logTags = ExternalLogger.tagString(tags)
 
     if (span) {

package/packages/dd-trace/src/opentelemetry/span.js

@@ -38,6 +38,7 @@ class Span {
       context: spanContext._ddContext,
       startTime,
       hostname: _tracer._hostname,
+      integrationName: 'otel',
       tags: {
         'service.name': _tracer._service
       }

package/packages/dd-trace/src/opentracing/span.js

@@ -11,6 +11,9 @@ const tagger = require('../tagger')
 const metrics = require('../metrics')
 const log = require('../log')
 const { storage } = require('../../../datadog-core')
+const telemetryMetrics = require('../telemetry/metrics')
+
+const tracerMetrics = telemetryMetrics.manager.namespace('tracers')
 
 const {
   DD_TRACE_EXPERIMENTAL_STATE_TRACKING,
@@ -20,6 +23,30 @@ const {
 const unfinishedRegistry = createRegistry('unfinished')
 const finishedRegistry = createRegistry('finished')
 
+const OTEL_ENABLED = !!process.env.DD_TRACE_OTEL_ENABLED
+
+const integrationCounters = {
+  span_created: {},
+  span_finished: {}
+}
+
+function getIntegrationCounter (event, integration) {
+  const counters = integrationCounters[event]
+
+  if (integration in counters) {
+    return counters[integration]
+  }
+
+  const counter = tracerMetrics.count(event, [
+    `integration_name:${integration.toLowerCase()}`,
+    `otel_enabled:${OTEL_ENABLED}`
+  ])
+
+  integrationCounters[event][integration] = counter
+
+  return counter
+}
+
 class DatadogSpan {
   constructor (tracer, processor, prioritySampler, fields, debug) {
     const operationName = fields.operationName
@@ -38,6 +65,9 @@ class DatadogSpan {
     // This name property is not updated when the span name changes.
     // This is necessary for span count metrics.
     this._name = operationName
+    this._integrationName = fields.integrationName || 'opentracing'
+
+    getIntegrationCounter('span_created', this._integrationName).inc()
 
     this._spanContext = this._createContext(parent, fields)
     this._spanContext._name = operationName
@@ -126,6 +156,8 @@ class DatadogSpan {
       }
     }
 
+    getIntegrationCounter('span_finished', this._integrationName).inc()
+
     if (DD_TRACE_EXPERIMENTAL_SPAN_COUNTS && finishedRegistry) {
       metrics.decrement('runtime.node.spans.unfinished')
       metrics.decrement('runtime.node.spans.unfinished.by.name', `span_name:${this._name}`)

package/packages/dd-trace/src/opentracing/tracer.js

@@ -26,6 +26,7 @@ class DatadogTracer {
     this._version = config.version
     this._env = config.env
     this._tags = config.tags
+    this._computePeerService = config.spanComputePeerService
     this._logInjection = config.logInjection
     this._debug = config.debug
     this._prioritySampler = new PrioritySampler(config.env, config.sampler)
@@ -60,7 +61,8 @@ class DatadogTracer {
       tags,
       startTime: options.startTime,
       hostname: this._hostname,
-      traceId128BitGenerationEnabled: this._traceId128BitGenerationEnabled
+      traceId128BitGenerationEnabled: this._traceId128BitGenerationEnabled,
+      integrationName: options.integrationName
     }, this._debug)
 
     span.addTags(this._tags)

package/packages/dd-trace/src/plugin_manager.js

@@ -21,7 +21,7 @@ const disabledPlugins = new Set(
   DD_TRACE_DISABLED_PLUGINS && DD_TRACE_DISABLED_PLUGINS.split(',').map(plugin => plugin.trim())
 )
 
-// TODO actually ... should we be looking at envrionment variables this deep down in the code?
+// TODO actually ... should we be looking at environment variables this deep down in the code?
 
 const pluginClasses = {}
 
@@ -133,7 +133,9 @@ module.exports = class PluginManager {
       serviceMapping,
       queryStringObfuscation,
       site,
-      url
+      url,
+      dbmPropagationMode,
+      dsmEnabled
     } = this._tracerConfig
 
     const sharedConfig = {}
@@ -146,6 +148,9 @@ module.exports = class PluginManager {
       sharedConfig.queryStringObfuscation = queryStringObfuscation
     }
 
+    sharedConfig.dbmPropagationMode = dbmPropagationMode
+    sharedConfig.dsmEnabled = dsmEnabled
+
     if (serviceMapping && serviceMapping[name]) {
       sharedConfig.service = serviceMapping[name]
     }

package/packages/dd-trace/src/plugins/client.js

@@ -5,6 +5,7 @@ const OutboundPlugin = require('./outbound')
 class ClientPlugin extends OutboundPlugin {
   static get operation () { return 'request' }
   static get kind () { return 'client' }
+  static get type () { return 'web' } // overridden by storage and other client type plugins
 }
 
 module.exports = ClientPlugin

package/packages/dd-trace/src/plugins/database.js

@@ -4,6 +4,7 @@ const StoragePlugin = require('./storage')
 
 class DatabasePlugin extends StoragePlugin {
   static get operation () { return 'query' }
+  static get peerServicePrecursors () { return ['db.name'] }
 
   constructor (...args) {
     super(...args)
@@ -38,7 +39,7 @@ class DatabasePlugin extends StoragePlugin {
   }
 
   injectDbmQuery (query, serviceName, isPreparedStatement = false) {
-    const mode = this.config.dbmPropagationMode || this._tracerConfig.dbmPropagationMode
+    const mode = this.config.dbmPropagationMode
 
     if (mode === 'disabled') {
       return query

package/packages/dd-trace/src/plugins/index.js

@@ -14,6 +14,7 @@ module.exports = {
   get '@node-redis/client' () { return require('../../../datadog-plugin-redis/src') },
   get '@opensearch-project/opensearch' () { return require('../../../datadog-plugin-opensearch/src') },
   get '@redis/client' () { return require('../../../datadog-plugin-redis/src') },
+  get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') },
   get 'amqp10' () { return require('../../../datadog-plugin-amqp10/src') },
   get 'amqplib' () { return require('../../../datadog-plugin-amqplib/src') },
   get 'aws-sdk' () { return require('../../../datadog-plugin-aws-sdk/src') },
@@ -56,6 +57,7 @@ module.exports = {
   get 'net' () { return require('../../../datadog-plugin-net/src') },
   get 'next' () { return require('../../../datadog-plugin-next/src') },
   get 'oracledb' () { return require('../../../datadog-plugin-oracledb/src') },
+  get 'openai' () { return require('../../../datadog-plugin-openai/src') },
   get 'paperplane' () { return require('../../../datadog-plugin-paperplane/src') },
   get 'pg' () { return require('../../../datadog-plugin-pg/src') },
   get 'pino' () { return require('../../../datadog-plugin-pino/src') },

package/packages/dd-trace/src/plugins/outbound.js

@@ -1,10 +1,21 @@
 'use strict'
 
-const { CLIENT_PORT_KEY } = require('../constants')
+const {
+  CLIENT_PORT_KEY,
+  PEER_SERVICE_KEY,
+  PEER_SERVICE_SOURCE_KEY
+} = require('../constants')
 const TracingPlugin = require('./tracing')
 
+const COMMON_PEER_SVC_SOURCE_TAGS = [
+  'net.peer.name',
+  'out.host'
+]
+
 // TODO: Exit span on finish when AsyncResource instances are removed.
 class OutboundPlugin extends TracingPlugin {
+  static get peerServicePrecursors () { return [] }
+
   constructor (...args) {
     super(...args)
 
@@ -13,6 +24,53 @@ class OutboundPlugin extends TracingPlugin {
     })
   }
 
+  getPeerService (tags) {
+    /**
+     * Compute `peer.service` and associated metadata from available tags, based
+     * on defined precursor tags names.
+     *
+     * - The `peer.service` tag is set from the first precursor available (based on list ordering)
+     * - The `_dd.peer.service.source` tag is set from the precursor's name
+     * - If `peer.service` was defined _before_ we compute it (for example in custom instrumentation),
+     *   `_dd.peer.service.source`'s value is `peer.service`
+     */
+
+    if (tags['peer.service'] !== undefined) {
+      return { [PEER_SERVICE_SOURCE_KEY]: 'peer.service' }
+    }
+
+    const sourceTags = [
+      ...this.constructor.peerServicePrecursors,
+      ...COMMON_PEER_SVC_SOURCE_TAGS
+    ]
+
+    for (const sourceTag of sourceTags) {
+      if (tags[sourceTag]) {
+        return {
+          [PEER_SERVICE_KEY]: tags[sourceTag],
+          [PEER_SERVICE_SOURCE_KEY]: sourceTag
+        }
+      }
+    }
+    return {}
+  }
+
+  startSpan (name, options) {
+    const span = super.startSpan(name, options)
+    return span
+  }
+
+  finish () {
+    const span = this.activeSpan
+    if (this.tracer._computePeerService) {
+      const peerData = this.getPeerService(span.context()._tags)
+      if (peerData) {
+        span.addTags(peerData)
+      }
+    }
+    super.finish(...arguments)
+  }
+
   connect (url) {
     this.addHost(url.hostname, url.port)
   }

package/packages/dd-trace/src/plugins/server.js

@@ -4,6 +4,8 @@ const InboundPlugin = require('./inbound')
 
 class ServerPlugin extends InboundPlugin {
   static get operation () { return 'request' }
+  static get kind () { return 'server' }
+  static get type () { return 'web' } // a default that may eventually be overriden by nonweb servers
 }
 
 module.exports = ServerPlugin

package/packages/dd-trace/src/plugins/tracing.js

@@ -33,6 +33,9 @@ class TracingPlugin extends Plugin {
   }
 
   serviceName (...serviceArgs) {
+    if (Nomenclature.shouldUseConsistentServiceNaming) {
+      return Nomenclature.shortCircuitServiceName(this.config, ...serviceArgs)
+    }
     const { type, id, kind } = this.constructor
     return Nomenclature.serviceName(type, kind, id, ...serviceArgs)
   }
@@ -91,7 +94,8 @@ class TracingPlugin extends Plugin {
         'span.type': type,
         ...meta,
         ...metrics
-      }
+      },
+      integrationName: type
     })
 
     analyticsSampler.sample(span, this.config.measured)

package/packages/dd-trace/src/plugins/util/exec.js

@@ -1,9 +1,11 @@
 const cp = require('child_process')
+const log = require('../../log')
 
 const sanitizedExec = (cmd, flags, options = { stdio: 'pipe' }) => {
   try {
     return cp.execFileSync(cmd, flags, options).toString().replace(/(\r\n|\n|\r)/gm, '')
   } catch (e) {
+    log.error(e)
     return ''
   }
 }

package/packages/dd-trace/src/plugins/util/git.js

@@ -35,13 +35,41 @@ function isShallowRepository () {
   return sanitizedExec('git', ['rev-parse', '--is-shallow-repository']) === 'true'
 }
 
-function unshallowRepository () {
+function getGitVersion () {
+  const gitVersionString = sanitizedExec('git', ['version'])
+  const gitVersionMatches = gitVersionString.match(/git version (\d+)\.(\d+)\.(\d+)/)
   try {
-    sanitizedExec('git', ['config', 'remote.origin.partialclonefilter', '"blob:none"'])
-    sanitizedExec('git', ['fetch', '--shallow-since="1 month ago"', '--update-shallow', '--refetch'])
-  } catch (err) {
-    log.error(err)
+    return {
+      major: parseInt(gitVersionMatches[1]),
+      minor: parseInt(gitVersionMatches[2]),
+      patch: parseInt(gitVersionMatches[3])
+    }
+  } catch (e) {
+    return null
+  }
+}
+
+function unshallowRepository () {
+  const gitVersion = getGitVersion()
+  if (!gitVersion) {
+    log.warn('Git version could not be extracted, so git unshallow will not proceed')
+    return
   }
+  if (gitVersion.major < 2 || (gitVersion.major === 2 && gitVersion.minor < 27)) {
+    log.warn('Git version is <2.27, so git unshallow will not proceed')
+    return
+  }
+  const defaultRemoteName = sanitizedExec('git', ['config', '--default', 'origin', '--get', 'clone.defaultRemoteName'])
+  const revParseHead = sanitizedExec('git', ['rev-parse', 'HEAD'])
+  sanitizedExec('git', [
+    'fetch',
+    '--shallow-since="1 month ago"',
+    '--update-shallow',
+    '--filter=blob:none',
+    '--recurse-submodules=no',
+    defaultRemoteName,
+    revParseHead
+  ])
 }
 
 function getRepositoryUrl () {
@@ -55,12 +83,12 @@ function getLatestCommits () {
       .split('\n')
       .filter(commit => commit)
   } catch (err) {
-    log.error(err)
+    log.error(`Get latest commits failed: ${err.message}`)
     return []
   }
 }
 
-function getCommitsToUpload (commitsToExclude) {
+function getCommitsToUpload (commitsToExclude, commitsToInclude) {
   const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`)
 
   try {
@@ -72,15 +100,15 @@ function getCommitsToUpload (commitsToExclude) {
         '--no-object-names',
         '--filter=blob:none',
         '--since="1 month ago"',
-        'HEAD',
-        ...commitsToExcludeString
+        ...commitsToExcludeString,
+        ...commitsToInclude
       ],
       { stdio: 'pipe', maxBuffer: GIT_REV_LIST_MAX_BUFFER })
       .toString()
       .split('\n')
       .filter(commit => commit)
   } catch (err) {
-    log.error(err)
+    log.error(`Get commits to upload failed: ${err.message}`)
     return []
   }
 }