@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/span_stats.js

@@ -0,0 +1,227 @@
+'use strict'
+
+const os = require('node:os')
+const pkg = require('../../../package.json')
+
+const { LogCollapsingLowestDenseDDSketch } = require('../../../vendor/dist/@datadog/sketches-js')
+const {
+  MEASURED,
+  HTTP_STATUS_CODE,
+  HTTP_ENDPOINT,
+  HTTP_ROUTE,
+  HTTP_METHOD,
+} = require('../../../ext/tags')
+const { ORIGIN_KEY, TOP_LEVEL_KEY } = require('./constants')
+const { version } = require('./pkg')
+const processTags = require('./process-tags')
+
+const { SpanStatsExporter } = require('./exporters/span-stats')
+
+const {
+  DEFAULT_SPAN_NAME,
+  DEFAULT_SERVICE_NAME,
+} = require('./encode/tags-processors')
+
+class SpanAggStats {
+  constructor (aggKey) {
+    this.aggKey = aggKey
+    this.hits = 0
+    this.topLevelHits = 0
+    this.errors = 0
+    this.duration = 0
+    this.okDistribution = new LogCollapsingLowestDenseDDSketch()
+    this.errorDistribution = new LogCollapsingLowestDenseDDSketch()
+  }
+
+  record (span) {
+    const durationNs = span.duration
+    this.hits++
+    this.duration += durationNs
+
+    if (span.metrics[TOP_LEVEL_KEY]) {
+      this.topLevelHits++
+    }
+
+    if (span.error) {
+      this.errors++
+      this.errorDistribution.accept(durationNs)
+    } else {
+      this.okDistribution.accept(durationNs)
+    }
+  }
+
+  toJSON () {
+    const {
+      name,
+      service,
+      resource,
+      type,
+      statusCode,
+      synthetics,
+      method,
+      endpoint,
+    } = this.aggKey
+
+    return {
+      Name: name,
+      Service: service,
+      Resource: resource,
+      Type: type,
+      HTTPStatusCode: statusCode,
+      Synthetics: synthetics,
+      HTTPMethod: method,
+      HTTPEndpoint: endpoint,
+      Hits: this.hits,
+      TopLevelHits: this.topLevelHits,
+      Errors: this.errors,
+      Duration: this.duration,
+      OkSummary: this.okDistribution.toProto(), // TODO: custom proto encoding
+      ErrorSummary: this.errorDistribution.toProto(), // TODO: custom proto encoding
+    }
+  }
+}
+
+class SpanAggKey {
+  constructor (span) {
+    this.name = span.name || DEFAULT_SPAN_NAME
+    this.service = span.service || DEFAULT_SERVICE_NAME
+    this.resource = span.resource || ''
+    this.type = span.type || ''
+    this.statusCode = span.meta[HTTP_STATUS_CODE] || 0
+    this.synthetics = span.meta[ORIGIN_KEY] === 'synthetics'
+    this.endpoint = span.meta[HTTP_ROUTE] || span.meta[HTTP_ENDPOINT] || ''
+    this.method = span.meta[HTTP_METHOD] || ''
+  }
+
+  toString () {
+    return [
+      this.name,
+      this.service,
+      this.resource,
+      this.type,
+      this.statusCode,
+      this.synthetics,
+      this.method,
+      this.endpoint,
+    ].join(',')
+  }
+}
+
+class SpanBuckets extends Map {
+  forSpan (span) {
+    const aggKey = new SpanAggKey(span)
+    const key = aggKey.toString()
+
+    if (!this.has(key)) {
+      this.set(key, new SpanAggStats(aggKey))
+    }
+
+    return this.get(key)
+  }
+}
+
+class TimeBuckets extends Map {
+  forTime (time) {
+    if (!this.has(time)) {
+      this.set(time, new SpanBuckets())
+    }
+
+    return this.get(time)
+  }
+}
+
+class SpanStatsProcessor {
+  constructor ({
+    stats: {
+      enabled = false,
+      interval = 10,
+    },
+    hostname,
+    port,
+    url,
+    env,
+    tags,
+    version,
+  } = {}) {
+    this.exporter = new SpanStatsExporter({
+      hostname,
+      port,
+      tags,
+      url,
+    })
+    this.interval = interval
+    this.bucketSizeNs = interval * 1e9
+    this.buckets = new TimeBuckets()
+    this.hostname = os.hostname()
+    this.enabled = enabled
+    this.env = env
+    this.tags = tags || {}
+    this.sequence = 0
+    this.version = version
+
+    if (this.enabled) {
+      this.timer = setInterval(this.onInterval.bind(this), interval * 1e3)
+      this.timer.unref()
+    }
+  }
+
+  onInterval () {
+    const serialized = this._serializeBuckets()
+    if (!serialized) return
+
+    this.exporter.export({
+      Hostname: this.hostname,
+      Env: this.env,
+      Version: this.version || version,
+      Stats: serialized,
+      Lang: 'javascript',
+      TracerVersion: pkg.version,
+      RuntimeID: this.tags['runtime-id'],
+      Sequence: ++this.sequence,
+      ProcessTags: processTags.serialized,
+    })
+  }
+
+  onSpanFinished (span) {
+    if (!this.enabled) return
+    if (!span.metrics[TOP_LEVEL_KEY] && !span.metrics[MEASURED]) return
+
+    const spanEndNs = span.startTime + span.duration
+    const bucketTime = spanEndNs - (spanEndNs % this.bucketSizeNs)
+
+    this.buckets.forTime(bucketTime)
+      .forSpan(span)
+      .record(span)
+  }
+
+  _serializeBuckets () {
+    const { bucketSizeNs } = this
+    const serializedBuckets = []
+
+    for (const [timeNs, bucket] of this.buckets.entries()) {
+      const bucketAggStats = []
+
+      for (const stats of bucket.values()) {
+        bucketAggStats.push(stats.toJSON())
+      }
+
+      serializedBuckets.push({
+        Start: timeNs,
+        Duration: bucketSizeNs,
+        Stats: bucketAggStats,
+      })
+    }
+
+    this.buckets.clear()
+
+    return serializedBuckets
+  }
+}
+
+module.exports = {
+  SpanAggStats,
+  SpanAggKey,
+  SpanBuckets,
+  TimeBuckets,
+  SpanStatsProcessor,
+}

package/packages/dd-trace/src/spanleak.js

@@ -0,0 +1,97 @@
+'use strict'
+
+/* eslint-disable no-console */
+
+const SortedSet = require('../../../vendor/dist/tlhunter-sorted-set')
+
+const INTERVAL = 1000 // look for expired spans every 1s
+const LIFETIME = 60 * 1000 // all spans have a max lifetime of 1m
+
+const MODES = {
+  DISABLED: 0,
+  // METRICS_ONLY
+  LOG: 1,
+  GC_AND_LOG: 2,
+  // GC
+}
+
+module.exports.MODES = MODES
+
+const spans = new SortedSet()
+
+// TODO: should these also be delivered as runtime metrics?
+
+// const registry = new FinalizationRegistry(name => {
+//   spans.del(span) // there is no span
+// })
+
+let interval
+let mode = MODES.DISABLED
+
+module.exports.disable = function () {
+  mode = MODES.DISABLED
+}
+
+module.exports.enableLogging = function () {
+  mode = MODES.LOG
+}
+
+module.exports.enableGarbageCollection = function () {
+  mode = MODES.GC_AND_LOG
+}
+
+module.exports.startScrubber = function () {
+  if (!isEnabled()) return
+
+  interval = setInterval(() => {
+    const now = Date.now()
+    const expired = spans.rangeByScore(0, now)
+
+    if (!expired.length) return
+
+    const gc = isGarbageCollecting()
+
+    const expirationsByType = Object.create(null) // { [spanType]: count }
+
+    for (const wrapped of expired) {
+      spans.del(wrapped)
+      const span = wrapped.deref()
+
+      if (!span) continue // span has already been garbage collected
+
+      // TODO: Should we also do things like record the route to help users debug leaks?
+      if (!expirationsByType[span._name]) expirationsByType[span._name] = 0
+      expirationsByType[span._name]++
+
+      if (!gc) continue // everything after this point is related to manual GC
+
+      // TODO: what else can we do to alleviate memory usage
+      span.context()._tags = Object.create(null)
+    }
+
+    console.log('expired spans:' +
+      Object.keys(expirationsByType).reduce((a, c) => `${a} ${c}: ${expirationsByType[c]}`, ''))
+  }, INTERVAL)
+}
+
+module.exports.stopScrubber = function () {
+  clearInterval(interval)
+}
+
+module.exports.addSpan = function (span) {
+  if (!isEnabled()) return
+
+  const now = Date.now()
+  const expiration = now + LIFETIME
+  const wrapped = new WeakRef(span)
+  spans.add(wrapped, expiration)
+  // registry.register(span, span._name)
+}
+
+function isEnabled () {
+  return mode > MODES.DISABLED
+}
+
+function isGarbageCollecting () {
+  return mode >= MODES.GC_AND_LOG
+}

package/packages/dd-trace/src/standalone/index.js

@@ -0,0 +1,70 @@
+'use strict'
+
+const { channel } = require('dc-polyfill')
+const { USER_KEEP } = require('../../../../ext/priority')
+const TraceState = require('../opentracing/propagation/tracestate')
+const { APM_TRACING_ENABLED_KEY } = require('../constants')
+const TraceSourcePrioritySampler = require('./tracesource_priority_sampler')
+const { hasTraceSourcePropagationTag } = require('./tracesource')
+
+const startCh = channel('dd-trace:span:start')
+const injectCh = channel('dd-trace:span:inject')
+const extractCh = channel('dd-trace:span:extract')
+
+function configure (config) {
+  if (startCh.hasSubscribers) startCh.unsubscribe(onSpanStart)
+  if (injectCh.hasSubscribers) injectCh.unsubscribe(onSpanInject)
+  if (extractCh.hasSubscribers) extractCh.unsubscribe(onSpanExtract)
+
+  if (config.apmTracingEnabled !== false) return
+
+  startCh.subscribe(onSpanStart)
+  injectCh.subscribe(onSpanInject)
+  extractCh.subscribe(onSpanExtract)
+
+  return new TraceSourcePrioritySampler(config.env)
+}
+
+function onSpanStart ({ span, fields }) {
+  const tags = span.context?.()?._tags
+  if (!tags) return
+
+  const { parent } = fields
+  if (!parent || parent._isRemote) {
+    tags[APM_TRACING_ENABLED_KEY] = 0
+  }
+}
+
+function onSpanInject ({ spanContext, carrier }) {
+  if (!spanContext?._trace?.tags || !carrier) return
+
+  // do not inject trace and sampling if there is no _dd.p.ts
+  if (!hasTraceSourcePropagationTag(spanContext._trace.tags)) {
+    for (const key in carrier) {
+      const lKey = key.toLowerCase()
+      if (lKey.startsWith('x-datadog') || lKey.startsWith('x-b3') || lKey === 'traceparent') {
+        delete carrier[key]
+      } else if (lKey === 'tracestate') {
+        const tracestate = TraceState.fromString(carrier[key])
+        tracestate.forVendor('dd', state => state.clear())
+        carrier[key] = tracestate.toString()
+      }
+    }
+  }
+}
+
+function onSpanExtract ({ spanContext = {} }) {
+  if (!spanContext._trace?.tags || !spanContext._sampling) return
+
+  // reset upstream priority if _dd.p.ts is not found
+  if (!hasTraceSourcePropagationTag(spanContext._trace.tags)) {
+    spanContext._sampling.priority = undefined
+  } else if (spanContext._sampling.priority !== USER_KEEP) {
+    spanContext._sampling.priority = USER_KEEP
+  }
+}
+
+module.exports = {
+  configure,
+  hasTraceSourcePropagationTag,
+}

package/packages/dd-trace/src/standalone/product.js

@@ -0,0 +1,36 @@
+'use strict'
+
+const { SAMPLING_MECHANISM_APPSEC, SAMPLING_MECHANISM_AI_GUARD } = require('../constants')
+const RateLimiter = require('../rate_limiter')
+
+/**
+ * Returns a rate limiter tuned for the provided product configuration.
+ *
+ * @param {{ appsec?: { enabled?: boolean }, iast?: { enabled?: boolean } } | undefined} config
+ * @returns {import('../rate_limiter')}
+ */
+function getProductRateLimiter (config) {
+  if (config?.appsec?.enabled || config?.iast?.enabled) {
+    return new RateLimiter(1, 'minute') // onePerMinute
+  }
+
+  return new RateLimiter(0) // dropAll
+}
+
+/**
+ * Available products and their identifiers/mechanisms.
+ */
+const PRODUCTS = {
+  APM: { id: 1 << 0 },
+  ASM: { id: 1 << 1, mechanism: SAMPLING_MECHANISM_APPSEC },
+  DSM: { id: 1 << 2 },
+  DJM: { id: 1 << 3 },
+  DBM: { id: 1 << 4 },
+  AI_GUARD: { id: 1 << 5, mechanism: SAMPLING_MECHANISM_AI_GUARD },
+}
+
+module.exports = {
+  ...PRODUCTS,
+
+  getProductRateLimiter,
+}

package/packages/dd-trace/src/standalone/tracesource.js

@@ -0,0 +1,36 @@
+'use strict'
+
+const { TRACE_SOURCE_PROPAGATION_KEY } = require('../constants')
+
+/**
+ * Adds or updates the trace source propagation tag with the given product bit.
+ *
+ * @param {Record<string, unknown>|undefined} tags
+ * @param {{ id: number }|undefined} product
+ * @returns {Record<string, unknown>|undefined}
+ */
+function addTraceSourceTag (tags, product) {
+  if (tags && product) {
+    const actual = tags[TRACE_SOURCE_PROPAGATION_KEY]
+      ? Number.parseInt(String(tags[TRACE_SOURCE_PROPAGATION_KEY]), 16)
+      : 0
+    tags[TRACE_SOURCE_PROPAGATION_KEY] = ((actual | product.id) >>> 0).toString(16).padStart(2, '0')
+  }
+
+  return tags
+}
+
+/**
+ * Returns true when the trace source propagation tag exists on the given tags object.
+ *
+ * @param {Record<string, unknown>} tags
+ * @returns {boolean}
+ */
+function hasTraceSourcePropagationTag (tags) {
+  return Object.hasOwn(tags, TRACE_SOURCE_PROPAGATION_KEY)
+}
+
+module.exports = {
+  addTraceSourceTag,
+  hasTraceSourcePropagationTag,
+}

package/packages/dd-trace/src/standalone/tracesource_priority_sampler.js

@@ -0,0 +1,59 @@
+'use strict'
+
+const PrioritySampler = require('../priority_sampler')
+const { MANUAL_KEEP } = require('../../../../ext/tags')
+const { USER_KEEP, AUTO_KEEP, AUTO_REJECT } = require('../../../../ext/priority')
+const { SAMPLING_MECHANISM_DEFAULT } = require('../constants')
+const { addTraceSourceTag, hasTraceSourcePropagationTag } = require('./tracesource')
+const { getProductRateLimiter } = require('./product')
+
+class TraceSourcePrioritySampler extends PrioritySampler {
+  /**
+   * @override
+   */
+  configure (env, sampler, config) {
+    // rules not supported
+    this._env = env
+    this._limiter = getProductRateLimiter(config)
+  }
+
+  /**
+   * @override
+   * @returns {import('../priority_sampler').SamplingPriority|undefined}
+   */
+  _getPriorityFromTags (tags, context) {
+    if (Object.hasOwn(tags, MANUAL_KEEP) &&
+      tags[MANUAL_KEEP] !== false &&
+      hasTraceSourcePropagationTag(context._trace.tags)
+    ) {
+      return USER_KEEP
+    }
+  }
+
+  /**
+   * @override
+   */
+  _getPriorityFromAuto (span) {
+    const context = this._getContext(span)
+
+    context._sampling.mechanism = SAMPLING_MECHANISM_DEFAULT
+
+    if (hasTraceSourcePropagationTag(context._trace.tags)) {
+      return USER_KEEP
+    }
+
+    return this._isSampledByRateLimit(context) ? AUTO_KEEP : AUTO_REJECT
+  }
+
+  /**
+   * @override
+   */
+  setPriority (span, samplingPriority, product) {
+    super.setPriority(span, samplingPriority, product)
+
+    const context = this._getContext(span)
+    addTraceSourceTag(context?._trace?.tags, product)
+  }
+}
+
+module.exports = TraceSourcePrioritySampler

package/packages/dd-trace/src/startup-log.js

@@ -0,0 +1,146 @@
+'use strict'
+
+const os = require('os')
+const { inspect } = require('util')
+const tracerVersion = require('../../../package.json').version
+const { getAgentUrl } = require('./agent/url')
+const { warn } = require('./log/writer')
+
+const errors = {}
+let config
+let pluginManager
+/** @type {import('./sampling_rule')[]} */
+let samplingRules = []
+let configAlreadyRan = false
+let integrationsAlreadyRan = false
+let agentErrorAlreadyRan = false
+
+/**
+ * Logs DATADOG TRACER CONFIGURATION immediately at init time.
+ * Excludes integrations_loaded since plugins haven't loaded yet.
+ */
+function startupLog () {
+  if (configAlreadyRan || !config || !config.startupLogs) {
+    return
+  }
+
+  configAlreadyRan = true
+
+  const out = configInfo()
+
+  warn('DATADOG TRACER CONFIGURATION - ' + out)
+}
+
+/**
+ * Logs loaded integrations. Called from writer.js on first agent payload,
+ * by which time the app has loaded its dependencies.
+ */
+function logIntegrations () {
+  if (integrationsAlreadyRan || !config || !config.startupLogs || !pluginManager) {
+    return
+  }
+
+  integrationsAlreadyRan = true
+
+  warn('DATADOG TRACER INTEGRATIONS LOADED - ' + JSON.stringify(Object.keys(pluginManager._pluginsByName)))
+}
+
+/**
+ * Logs agent error diagnostic.
+ * @param {{ status: number, message: string }} agentError
+ */
+function logAgentError (agentError) {
+  if (agentErrorAlreadyRan || !config || !config.startupLogs) {
+    return
+  }
+
+  agentErrorAlreadyRan = true
+
+  warn('DATADOG TRACER DIAGNOSTIC - Agent Error: ' + agentError.message)
+  errors.agentError = {
+    code: agentError.status,
+    message: `Agent Error: ${agentError.message}`,
+  }
+}
+
+/**
+ * Returns config info without integrations (used by startupLog).
+ * @returns {Record<string, unknown>}
+ */
+function configInfo () {
+  const url = getAgentUrl(config)
+
+  return {
+    [inspect.custom] () {
+      return String(this)
+    },
+    toString () {
+      return JSON.stringify(this, (_key_, value) => {
+        return typeof value === 'bigint' || typeof value === 'symbol' ? String(value) : value
+      })
+    },
+    date: new Date().toISOString(),
+    os_name: os.type(),
+    os_version: os.release(),
+    architecture: os.arch(),
+    version: tracerVersion,
+    lang: 'nodejs',
+    lang_version: process.versions.node,
+    env: config.env,
+    enabled: config.enabled,
+    service: config.service,
+    agent_url: url,
+    debug: !!config.debug,
+    sample_rate: config.sampler.sampleRate,
+    sampling_rules: samplingRules,
+    tags: config.tags,
+    ...(config.tags && config.tags.version && { dd_version: config.tags.version }),
+    log_injection_enabled: !!config.logInjection,
+    runtime_metrics_enabled: !!config.runtimeMetrics,
+    profiling_enabled: config.profiling?.enabled === 'true' || config.profiling?.enabled === 'auto',
+    appsec_enabled: config.appsec.enabled,
+    data_streams_enabled: !!config.dsmEnabled,
+  }
+}
+
+/**
+ * Returns full tracer info including integrations (used by flare module).
+ * @returns {Record<string, unknown>}
+ */
+function tracerInfo () {
+  const out = configInfo()
+  out.integrations_loaded = Object.keys(pluginManager._pluginsByName)
+  return out
+}
+
+/**
+ * @param {import('./config')} aConfig
+ */
+function setStartupLogConfig (aConfig) {
+  config = aConfig
+}
+
+/**
+ * @param {import('./plugin_manager')} thePluginManager
+ */
+function setStartupLogPluginManager (thePluginManager) {
+  pluginManager = thePluginManager
+}
+
+/**
+ * @param {import('./sampling_rule')[]} theRules
+ */
+function setSamplingRules (theRules) {
+  samplingRules = theRules
+}
+
+module.exports = {
+  startupLog,
+  logIntegrations,
+  logAgentError,
+  setStartupLogConfig,
+  setStartupLogPluginManager,
+  setSamplingRules,
+  tracerInfo,
+  errors,
+}