@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/telemetry/metrics.js

@@ -0,0 +1,288 @@
+'use strict'
+
+const { sendData } = require('./send-data')
+
+function getId (type, namespace, name, tags) {
+  return `${type}:${namespace}.${name}:${tagArray(tags).sort().join(',')}`
+}
+
+function tagArray (tags = {}) {
+  if (Array.isArray(tags)) return tags
+  const list = []
+  for (const [key, value] of Object.entries(tags)) {
+    list.push(`${key}:${value}`.toLowerCase())
+  }
+  return list
+}
+
+function now () {
+  return Date.now() / 1e3
+}
+
+function mapToJsonArray (map, filter) {
+  const array = []
+  for (const value of map.values()) {
+    if (!filter || filter(value)) {
+      array.push(value.toJSON())
+    }
+  }
+  return array
+}
+
+function hasPoints (metric) {
+  return metric.points.length > 0
+}
+
+class Metric {
+  constructor (namespace, metric, common, tags) {
+    this.namespace = namespace.toString()
+    this.metric = common ? metric : `nodejs.${metric}`
+    this.tags = tagArray(tags)
+    this.common = common
+
+    this.points = []
+  }
+
+  toString () {
+    const { namespace, metric } = this
+    return `${namespace}.${metric}`
+  }
+
+  reset () {
+    this.points = []
+  }
+
+  track () {
+    throw new Error('not implemented')
+  }
+
+  toJSON () {
+    const { metric, points, interval, type, tags, common } = this
+    return {
+      metric,
+      points,
+      interval,
+      type,
+      tags,
+      common,
+    }
+  }
+}
+
+class CountMetric extends Metric {
+  get type () {
+    return 'count'
+  }
+
+  inc (value) {
+    return this.track(value)
+  }
+
+  dec (value = 1) {
+    return this.track(-value)
+  }
+
+  track (value = 1) {
+    if (this.points.length) {
+      this.points[0][1] += value
+    } else {
+      this.points.push([now(), value])
+    }
+  }
+}
+
+class DistributionMetric extends Metric {
+  get type () {
+    return 'distribution'
+  }
+
+  track (value = 1) {
+    this.points.push(value)
+  }
+
+  toJSON () {
+    const { metric, points, tags, common } = this
+    return {
+      metric,
+      points,
+      common,
+      tags,
+    }
+  }
+}
+
+class GaugeMetric extends Metric {
+  get type () {
+    return 'gauge'
+  }
+
+  mark (value) {
+    return this.track(value)
+  }
+
+  track (value = 1) {
+    this.points.push([now(), value])
+  }
+}
+
+class RateMetric extends Metric {
+  constructor (namespace, metric, common, tags, interval) {
+    super(namespace, metric, common, tags)
+
+    this.interval = interval
+    this.rate = 0
+  }
+
+  get type () {
+    return 'rate'
+  }
+
+  reset () {
+    super.reset()
+    this.rate = 0
+  }
+
+  track (value = 1) {
+    this.rate += value
+    const rate = this.interval ? (this.rate / this.interval) : 0
+    this.points = [[now(), rate]]
+  }
+}
+
+const metricsTypes = {
+  count: CountMetric,
+  distribution: DistributionMetric,
+  gauge: GaugeMetric,
+  rate: RateMetric,
+}
+
+class MetricsCollection extends Map {
+  constructor (namespace) {
+    super()
+    this.namespace = namespace
+  }
+
+  reset () {
+    for (const metric of this.values()) {
+      metric.reset()
+    }
+  }
+
+  toString () {
+    return this.namespace
+  }
+
+  toJSON () {
+    if (!this.size) return
+
+    const series = mapToJsonArray(this, hasPoints)
+
+    if (!series.length) return
+
+    const { namespace } = this
+    return {
+      namespace,
+      series,
+    }
+  }
+}
+
+function getMetric (collection, type, name, tags, interval) {
+  const metricId = getId(type, collection, name, tags)
+
+  let metric = collection.get(metricId)
+  if (metric) return metric
+
+  const Factory = metricsTypes[type]
+  if (!Factory) {
+    throw new Error(`Unknown metric type ${type}`)
+  }
+
+  metric = new Factory(collection, name, true, tags, interval)
+  collection.set(metricId, metric)
+
+  return metric
+}
+
+class Namespace {
+  constructor (namespace) {
+    this.distributions = new MetricsCollection(namespace)
+    this.metrics = new MetricsCollection(namespace)
+  }
+
+  reset () {
+    this.metrics.reset()
+    this.distributions.reset()
+  }
+
+  count (name, tags) {
+    return getMetric(this.metrics, 'count', name, tags)
+  }
+
+  gauge (name, tags) {
+    return getMetric(this.metrics, 'gauge', name, tags)
+  }
+
+  rate (name, interval, tags) {
+    return getMetric(this.metrics, 'rate', name, tags, interval)
+  }
+
+  distribution (name, tags) {
+    return getMetric(this.distributions, 'distribution', name, tags)
+  }
+
+  toJSON () {
+    const { distributions, metrics } = this
+    return {
+      distributions: distributions.toJSON(),
+      metrics: metrics.toJSON(),
+    }
+  }
+}
+
+class NamespaceManager extends Map {
+  namespace (name) {
+    let ns = this.get(name)
+    if (ns) return ns
+
+    ns = new Namespace(name)
+    this.set(name, ns)
+    return ns
+  }
+
+  toJSON () {
+    return mapToJsonArray(this)
+  }
+
+  send (config, application, host) {
+    for (const namespace of this.values()) {
+      const { metrics, distributions } = namespace.toJSON()
+
+      if (metrics) {
+        sendData(config, application, host, 'generate-metrics', metrics)
+      }
+
+      if (distributions) {
+        sendData(config, application, host, 'distributions', distributions)
+      }
+
+      // TODO: This could also be clear() but then it'd have to rebuild all
+      // metric instances on every send. This may be desirable if we want tags
+      // with high cardinality and variability over time.
+      namespace.reset()
+    }
+  }
+}
+
+const manager = new NamespaceManager()
+
+module.exports = {
+  CountMetric,
+  DistributionMetric,
+  GaugeMetric,
+  RateMetric,
+  MetricsCollection,
+  Namespace,
+  NamespaceManager,
+  manager,
+}

package/packages/dd-trace/src/telemetry/send-data.js

@@ -0,0 +1,225 @@
+'use strict'
+
+const request = require('../exporters/common/request')
+const log = require('../log')
+const { isTrue } = require('../util')
+const { getValueFromEnvSources } = require('../config/helper')
+
+/**
+ * @typedef {Record<string, unknown>} TelemetryPayloadObject
+ */
+/**
+ * Telemetry "request_type" values sent by this library.
+ *
+ * @typedef {'app-started'
+ *   | 'app-integrations-change'
+ *   | 'app-heartbeat'
+ *   | 'app-extended-heartbeat'
+ *   | 'app-client-configuration-change'
+ *   | 'app-closing'
+ *   | 'app-dependencies-loaded'
+ *   | 'app-endpoints'
+ *   | 'generate-metrics'
+ *   | 'distributions'
+ *   | 'logs'
+ *   | 'message-batch'} TelemetryRequestType
+ */
+/**
+ * @typedef {{ request_type: string, payload: TelemetryPayloadObject }} MessageBatchItem
+ */
+/**
+ * @typedef {MessageBatchItem[]} MessageBatchPayload
+ */
+/**
+ * Telemetry payloads are usually single objects, but some request types (e.g. `message-batch`)
+ * send arrays.
+ *
+ * @typedef {TelemetryPayloadObject | MessageBatchPayload} TelemetryPayload
+ */
+/**
+ * @typedef {{language_name: string, tracer_version: string} & Record<string, unknown>} TelemetryApplication
+ */
+/**
+ * @typedef {Exclude<TelemetryRequestType, 'message-batch'>} NonBatchTelemetryRequestType
+ */
+/**
+ * @typedef {{
+ *   payload: TelemetryPayloadObject,
+ *   reqType: NonBatchTelemetryRequestType
+ * } | {
+ *   payload: MessageBatchPayload,
+ *   reqType: 'message-batch'
+ * }} SendDataRetryObject
+ */
+/**
+ * @typedef {{
+ *   hostname: string,
+ *   os: string,
+ *   architecture: string,
+ *   os_version?: string,
+ *   kernel_version?: string,
+ *   kernel_release?: string,
+ *   kernel_name?: string
+ * } & Record<string, unknown>} TelemetryHost
+ */
+/**
+ * @typedef {{
+ *   hostname?: string,
+ *   port?: string | number,
+ *   url?: string | URL,
+ *   site?: string,
+ *   apiKey?: string,
+ *   isCiVisibility?: boolean,
+ *   spanAttributeSchema?: string,
+ *   tags: Record<string, string>,
+ *   telemetry?: { debug?: boolean }
+ * }} TelemetryConfig
+ */
+/**
+ * @callback SendDataCallback
+ * @param {Error | null | undefined} error
+ * @param {SendDataRetryObject} retryObj
+ * @returns {void}
+ */
+
+let agentTelemetry = true
+
+/**
+ * @param {TelemetryConfig} config
+ * @param {TelemetryApplication} application
+ * @param {TelemetryRequestType} reqType
+ * @returns {Record<string, string>}
+ */
+function getHeaders (config, application, reqType) {
+  const headers = {
+    'content-type': 'application/json',
+    'dd-telemetry-api-version': 'v2',
+    'dd-telemetry-request-type': reqType,
+    'dd-client-library-language': application.language_name,
+    'dd-client-library-version': application.tracer_version,
+  }
+  const debug = config.telemetry && config.telemetry.debug
+  if (debug) {
+    headers['dd-telemetry-debug-enabled'] = 'true'
+  }
+  if (config.apiKey) {
+    headers['dd-api-key'] = config.apiKey
+  }
+  return headers
+}
+
+/**
+ * @param {string | undefined} site
+ */
+function getAgentlessTelemetryEndpoint (site) {
+  if (site === 'datad0g.com') { // staging
+    return 'https://all-http-intake.logs.datad0g.com'
+  }
+  return `https://instrumentation-telemetry-intake.${site}`
+}
+
+let seqId = 0
+
+/**
+ * @param {TelemetryPayload} payload
+ * @returns {TelemetryPayload}
+ */
+function getPayload (payload) {
+  // Some telemetry endpoints payloads accept collections of elements such as the 'logs' endpoint.
+  // 'logs' request type payload is meant to send library logs to Datadog’s backend.
+  if (Array.isArray(payload)) {
+    return payload
+  }
+  const { logger, tags, serviceMapping, ...trimmedPayload } = payload
+  return trimmedPayload
+}
+
+// TODO(BridgeAR): Simplify this code. A lot does not need to be recalculated on every call.
+/**
+ * @param {TelemetryConfig} config
+ * @param {TelemetryApplication} application
+ * @param {TelemetryHost} host
+ * @param {TelemetryRequestType} reqType
+ * @param {TelemetryPayload} [payload]
+ * @param {SendDataCallback} [cb]
+ */
+function sendData (config, application, host, reqType, payload = {}, cb = () => {}) {
+  const {
+    hostname,
+    port,
+    isCiVisibility,
+  } = config
+
+  let url = config.url
+
+  const isCiVisibilityAgentlessMode = isCiVisibility &&
+                                      isTrue(getValueFromEnvSources('DD_CIVISIBILITY_AGENTLESS_ENABLED'))
+
+  if (isCiVisibilityAgentlessMode) {
+    try {
+      url = url || new URL(getAgentlessTelemetryEndpoint(config.site))
+    } catch (err) {
+      log.error('Telemetry endpoint url is invalid', err)
+      // No point to do the request if the URL is invalid
+      return cb(err, { payload, reqType })
+    }
+  }
+
+  const options = {
+    url,
+    hostname,
+    port,
+    method: 'POST',
+    path: isCiVisibilityAgentlessMode ? '/api/v2/apmtelemetry' : '/telemetry/proxy/api/v2/apmtelemetry',
+    headers: getHeaders(config, application, reqType),
+  }
+
+  const data = JSON.stringify({
+    api_version: 'v2',
+    naming_schema_version: config.spanAttributeSchema ?? '',
+    request_type: reqType,
+    tracer_time: Math.floor(Date.now() / 1000),
+    runtime_id: config.tags['runtime-id'],
+    seq_id: ++seqId,
+    payload: getPayload(payload),
+    application,
+    host,
+  })
+
+  request(data, options, (error) => {
+    if (error && getValueFromEnvSources('DD_API_KEY') && config.site) {
+      if (agentTelemetry) {
+        log.warn('Agent telemetry failed, started agentless telemetry')
+        agentTelemetry = false
+      }
+      // figure out which data center to send to
+      const backendUrl = getAgentlessTelemetryEndpoint(config.site)
+      const backendHeader = { ...options.headers, 'DD-API-KEY': getValueFromEnvSources('DD_API_KEY') }
+      const backendOptions = {
+        ...options,
+        url: backendUrl,
+        headers: backendHeader,
+        path: '/api/v2/apmtelemetry',
+      }
+      if (backendUrl) {
+        request(data, backendOptions, (error) => {
+          if (error) {
+            log.error('Error sending telemetry data', error)
+          }
+        })
+      } else {
+        log.error('Invalid Telemetry URL')
+      }
+    }
+
+    if (!error && !agentTelemetry) {
+      agentTelemetry = true
+      log.info('Started agent telemetry')
+    }
+
+    // call the callback function so that we can track the error and payload
+    cb(error, { payload, reqType })
+  })
+}
+
+module.exports = { sendData }