dd-trace 4.3.0 → 4.5.0

package/packages/dd-trace/src/plugins/util/user-provided-git.js

@@ -13,6 +13,8 @@ const {
 } = require('./tags')
 
 const { normalizeRef } = require('./ci')
+const log = require('../../log')
+const { URL } = require('url')
 
 function removeEmptyValues (tags) {
   return Object.keys(tags).reduce((filteredTags, tag) => {
@@ -39,6 +41,37 @@ function filterSensitiveInfoFromRepository (repositoryUrl) {
   }
 }
 
+// The regex is extracted from
+// https://github.com/jonschlinkert/is-git-url/blob/396965ffabf2f46656c8af4c47bef1d69f09292e/index.js#L9C15-L9C87
+function validateGitRepositoryUrl (repoUrl) {
+  return /(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|#[-\d\w._]+?)$/.test(repoUrl)
+}
+
+function validateGitCommitSha (gitCommitSha) {
+  const isValidSha1 = /^[0-9a-f]{40}$/.test(gitCommitSha)
+  const isValidSha256 = /^[0-9a-f]{64}$/.test(gitCommitSha)
+  return isValidSha1 || isValidSha256
+}
+
+function removeInvalidGitMetadata (metadata) {
+  return Object.keys(metadata).reduce((filteredTags, tag) => {
+    if (tag === GIT_REPOSITORY_URL) {
+      if (!validateGitRepositoryUrl(metadata[GIT_REPOSITORY_URL])) {
+        log.error('DD_GIT_REPOSITORY_URL must be a valid URL')
+        return filteredTags
+      }
+    }
+    if (tag === GIT_COMMIT_SHA) {
+      if (!validateGitCommitSha(metadata[GIT_COMMIT_SHA])) {
+        log.error('DD_GIT_COMMIT_SHA must be a full-length git SHA')
+        return filteredTags
+      }
+    }
+    filteredTags[tag] = metadata[tag]
+    return filteredTags
+  }, {})
+}
+
 function getUserProviderGitMetadata () {
   const {
     DD_GIT_COMMIT_SHA,
@@ -62,7 +95,7 @@ function getUserProviderGitMetadata () {
     tag = normalizeRef(DD_GIT_BRANCH)
   }
 
-  return removeEmptyValues({
+  const metadata = removeEmptyValues({
     [GIT_COMMIT_SHA]: DD_GIT_COMMIT_SHA,
     [GIT_BRANCH]: branch,
     [GIT_REPOSITORY_URL]: filterSensitiveInfoFromRepository(DD_GIT_REPOSITORY_URL),
@@ -75,6 +108,7 @@ function getUserProviderGitMetadata () {
     [GIT_COMMIT_AUTHOR_EMAIL]: DD_GIT_COMMIT_AUTHOR_EMAIL,
     [GIT_COMMIT_AUTHOR_DATE]: DD_GIT_COMMIT_AUTHOR_DATE
   })
+  return removeInvalidGitMetadata(metadata)
 }
 
-module.exports = { getUserProviderGitMetadata }
+module.exports = { getUserProviderGitMetadata, validateGitRepositoryUrl, validateGitCommitSha }

package/packages/dd-trace/src/profiling/config.js

@@ -12,7 +12,7 @@ const WallProfiler = require('./profilers/wall')
 const SpaceProfiler = require('./profilers/space')
 const { oomExportStrategies, snapshotKinds } = require('./constants')
 const { tagger } = require('./tagger')
-const { isTrue } = require('../util')
+const { isFalse, isTrue } = require('../util')
 
 class Config {
   constructor (options = {}) {
@@ -32,6 +32,8 @@ class Config {
       DD_PROFILING_SOURCE_MAP,
       DD_PROFILING_UPLOAD_PERIOD,
       DD_PROFILING_PPROF_PREFIX,
+      DD_PROFILING_HEAP_ENABLED,
+      DD_PROFILING_WALLTIME_ENABLED,
       DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED,
       DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE,
       DD_PROFILING_EXPERIMENTAL_OOM_MAX_HEAP_EXTENSION_COUNT,
@@ -53,7 +55,7 @@ class Config {
     const endpointCollection = coalesce(options.endpointCollection,
       DD_PROFILING_ENDPOINT_COLLECTION_ENABLED, false)
     const pprofPrefix = coalesce(options.pprofPrefix,
-      DD_PROFILING_PPROF_PREFIX)
+      DD_PROFILING_PPROF_PREFIX, '')
 
     this.enabled = enabled
     this.service = service
@@ -88,7 +90,7 @@ class Config {
     ], this)
 
     const oomMonitoringEnabled = isTrue(coalesce(options.oomMonitoring,
-      DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED, false))
+      DD_PROFILING_EXPERIMENTAL_OOM_MONITORING_ENABLED, true))
     const heapLimitExtensionSize = coalesce(options.oomHeapLimitExtensionSize,
       Number(DD_PROFILING_EXPERIMENTAL_OOM_HEAP_LIMIT_EXTENSION_SIZE), 0)
     const maxHeapExtensionCount = coalesce(options.oomMaxHeapExtensionCount,
@@ -106,10 +108,9 @@ class Config {
       exportCommand
     }
 
-    const profilers = coalesce(options.profilers, DD_PROFILING_PROFILERS, [
-      new WallProfiler(this),
-      new SpaceProfiler(this)
-    ])
+    const profilers = options.profilers
+      ? options.profilers
+      : getProfilers({ DD_PROFILING_HEAP_ENABLED, DD_PROFILING_WALLTIME_ENABLED, DD_PROFILING_PROFILERS })
 
     this.profilers = ensureProfilers(profilers, this)
   }
@@ -117,6 +118,32 @@ class Config {
 
 module.exports = { Config }
 
+function getProfilers ({ DD_PROFILING_HEAP_ENABLED, DD_PROFILING_WALLTIME_ENABLED, DD_PROFILING_PROFILERS }) {
+  // First consider "legacy" DD_PROFILING_PROFILERS env variable, defaulting to wall + space
+  // Use a Set to avoid duplicates
+  const profilers = new Set(coalesce(DD_PROFILING_PROFILERS, 'wall,space').split(','))
+
+  // Add/remove wall depending on the value of DD_PROFILING_WALLTIME_ENABLED
+  if (DD_PROFILING_WALLTIME_ENABLED != null) {
+    if (isTrue(DD_PROFILING_WALLTIME_ENABLED)) {
+      profilers.add('wall')
+    } else if (isFalse(DD_PROFILING_WALLTIME_ENABLED)) {
+      profilers.delete('wall')
+    }
+  }
+
+  // Add/remove wall depending on the value of DD_PROFILING_HEAP_ENABLED
+  if (DD_PROFILING_HEAP_ENABLED != null) {
+    if (isTrue(DD_PROFILING_HEAP_ENABLED)) {
+      profilers.add('space')
+    } else if (isFalse(DD_PROFILING_HEAP_ENABLED)) {
+      profilers.delete('space')
+    }
+  }
+
+  return [...profilers]
+}
+
 function getExportStrategy (name, options) {
   const strategy = Object.values(oomExportStrategies).find(value => value === name)
   if (strategy === undefined) {

package/packages/dd-trace/src/proxy.js

@@ -64,6 +64,12 @@ class Tracer extends NoopProxy {
         this._pluginManager.configure(config)
         setStartupLogPluginManager(this._pluginManager)
         telemetry.start(config, this._pluginManager)
+
+        if (config.isManualApiEnabled) {
+          const TestApiManualPlugin = require('./ci-visibility/test-api-manual/test-api-manual-plugin')
+          this._testApiManualPlugin = new TestApiManualPlugin(this)
+          this._testApiManualPlugin.configure({ ...config, enabled: true })
+        }
       }
     } catch (e) {
       log.error(e)

package/packages/dd-trace/src/service-naming/index.js

@@ -3,7 +3,7 @@ const { schemaDefinitions } = require('./schemas')
 class SchemaManager {
   constructor () {
     this.schemas = schemaDefinitions
-    this.config = { spanAttributeSchema: 'v0' }
+    this.config = { spanAttributeSchema: 'v0', traceRemoveIntegrationServiceNamesEnabled: false }
   }
 
   get schema () {
@@ -14,6 +14,10 @@ class SchemaManager {
     return this.config.spanAttributeSchema
   }
 
+  get shouldUseConsistentServiceNaming () {
+    return this.config.traceRemoveIntegrationServiceNamesEnabled && this.version === 'v0'
+  }
+
   opName (type, kind, plugin, ...opNameArgs) {
     return this.schema.getOpName(type, kind, plugin, ...opNameArgs)
   }
@@ -22,6 +26,14 @@ class SchemaManager {
     return this.schema.getServiceName(type, kind, plugin, this.config.service, ...serviceNameArgs)
   }
 
+  shortCircuitServiceName (pluginConfig, ...args) {
+    // We're short-circuiting, so we do not obey custom service functions
+    if (typeof pluginConfig.service === 'function') {
+      return this.config.service
+    }
+    return pluginConfig.service || this.config.service
+  }
+
   configure (config = {}) {
     this.config = config
   }

package/packages/dd-trace/src/service-naming/schemas/v0/index.js

@@ -1,5 +1,6 @@
 const SchemaDefinition = require('../definition')
 const messaging = require('./messaging')
 const storage = require('./storage')
+const web = require('./web')
 
-module.exports = new SchemaDefinition({ messaging, storage })
+module.exports = new SchemaDefinition({ messaging, storage, web })

package/packages/dd-trace/src/service-naming/schemas/v0/storage.js

@@ -16,7 +16,16 @@ function mysqlServiceName (service, config, dbConfig, system) {
   if (typeof config.service === 'function') {
     return config.service(dbConfig)
   }
-  return config.service ? config.service : fromSystem(service, system)
+  return config.service || fromSystem(service, system)
+}
+
+function withSuffixFunction (suffix) {
+  return (service, config, params) => {
+    if (typeof config.service === 'function') {
+      return config.service(params)
+    }
+    return config.service || `${service}-${suffix}`
+  }
 }
 
 const redisConfig = {
@@ -28,6 +37,14 @@ const redisConfig = {
 
 const storage = {
   client: {
+    'cassandra-driver': {
+      opName: () => 'cassandra.query',
+      serviceName: (service, config, system) => config.service || fromSystem(service, system)
+    },
+    elasticsearch: {
+      opName: () => 'elasticsearch.query',
+      serviceName: (service, config) => config.service || `${service}-elasticsearch`
+    },
     ioredis: redisConfig,
     mariadb: {
       opName: () => 'mariadb.query',
@@ -37,6 +54,10 @@ const storage = {
       opName: () => 'memcached.command',
       serviceName: (service, config, system) => config.service || fromSystem(service, system)
     },
+    'mongodb-core': {
+      opName: () => 'mongodb.query',
+      serviceName: (service, config) => config.service || `${service}-mongodb`
+    },
     mysql: {
       opName: () => 'mysql.query',
       serviceName: mysqlServiceName
@@ -45,6 +66,18 @@ const storage = {
       opName: () => 'mysql.query',
       serviceName: mysqlServiceName
     },
+    opensearch: {
+      opName: () => 'opensearch.query',
+      serviceName: (service, config) => config.service || `${service}-opensearch`
+    },
+    oracledb: {
+      opName: () => 'oracle.query',
+      serviceName: withSuffixFunction('oracle')
+    },
+    pg: {
+      opName: () => 'pg.query',
+      serviceName: withSuffixFunction('postgres')
+    },
     redis: redisConfig,
     tedious: {
       opName: () => 'tedious.request',

package/packages/dd-trace/src/service-naming/schemas/v0/web.js

@@ -0,0 +1,27 @@
+const { identityService } = require('../util')
+const { DD_MAJOR } = require('../../../../../../version')
+
+const web = {
+  client: {
+    grpc: {
+      opName: () => DD_MAJOR <= 2 ? 'grpc.request' : 'grpc.client',
+      serviceName: identityService
+    },
+    moleculer: {
+      opName: () => 'moleculer.call',
+      serviceName: identityService
+    }
+  },
+  server: {
+    grpc: {
+      opName: () => DD_MAJOR <= 2 ? 'grpc.request' : 'grpc.server',
+      serviceName: identityService
+    },
+    moleculer: {
+      opName: () => 'moleculer.action',
+      serviceName: identityService
+    }
+  }
+}
+
+module.exports = web

package/packages/dd-trace/src/service-naming/schemas/v1/index.js

@@ -1,5 +1,6 @@
 const SchemaDefinition = require('../definition')
 const messaging = require('./messaging')
 const storage = require('./storage')
+const web = require('./web')
 
-module.exports = new SchemaDefinition({ messaging, storage })
+module.exports = new SchemaDefinition({ messaging, storage, web })

package/packages/dd-trace/src/service-naming/schemas/v1/storage.js

@@ -14,8 +14,23 @@ const mySQLNaming = {
   serviceName: identityService
 }
 
+function withFunction (service, config, params) {
+  if (typeof config.service === 'function') {
+    return config.service(params)
+  }
+  return configWithFallback(service, config)
+}
+
 const storage = {
   client: {
+    'cassandra-driver': {
+      opName: () => 'cassandra.query',
+      serviceName: configWithFallback
+    },
+    elasticsearch: {
+      opName: () => 'elasticsearch.query',
+      serviceName: configWithFallback
+    },
     ioredis: redisNaming,
     mariadb: {
       opName: () => 'mariadb.query',
@@ -25,8 +40,24 @@ const storage = {
       opName: () => 'memcached.command',
       serviceName: configWithFallback
     },
+    'mongodb-core': {
+      opName: () => 'mongodb.query',
+      serviceName: configWithFallback
+    },
     mysql: mySQLNaming,
     mysql2: mySQLNaming,
+    opensearch: {
+      opName: () => 'opensearch.query',
+      serviceName: configWithFallback
+    },
+    oracledb: {
+      opName: () => 'oracle.query',
+      serviceName: withFunction
+    },
+    pg: {
+      opName: () => 'postgresql.query',
+      serviceName: withFunction
+    },
     redis: redisNaming,
     tedious: {
       opName: () => 'mssql.query',

package/packages/dd-trace/src/service-naming/schemas/v1/web.js

@@ -0,0 +1,26 @@
+const { identityService } = require('../util')
+
+const web = {
+  client: {
+    grpc: {
+      opName: () => 'grpc.client.request',
+      serviceName: identityService
+    },
+    moleculer: {
+      opName: () => 'moleculer.client.request',
+      serviceName: identityService
+    }
+  },
+  server: {
+    grpc: {
+      opName: () => 'grpc.server.request',
+      serviceName: identityService
+    },
+    moleculer: {
+      opName: () => 'moleculer.server.request',
+      serviceName: identityService
+    }
+  }
+}
+
+module.exports = web

package/packages/dd-trace/src/telemetry/index.js

@@ -6,6 +6,8 @@ const os = require('os')
 const dependencies = require('./dependencies')
 const { sendData } = require('./send-data')
 
+const { manager: metricsManager } = require('./metrics')
+
 const telemetryStartChannel = dc.channel('datadog:telemetry:start')
 const telemetryStopChannel = dc.channel('datadog:telemetry:stop')
 
@@ -121,6 +123,7 @@ function start (aConfig, thePluginManager) {
   dependencies.start(config, application, host)
   sendData(config, application, host, 'app-started', appStarted())
   interval = setInterval(() => {
+    metricsManager.send(config, application, host)
     sendData(config, application, host, 'app-heartbeat')
   }, heartbeatInterval)
   interval.unref()

package/packages/dd-trace/src/telemetry/metrics.js

@@ -0,0 +1,281 @@
+'use strict'
+
+const { version } = require('../../../../package.json')
+
+const { sendData } = require('./send-data')
+
+function getId (type, namespace, name, tags) {
+  return `${type}:${namespace}.${name}:${tagArray(tags).sort().join(',')}`
+}
+
+function tagArray (tags = {}) {
+  if (Array.isArray(tags)) return tags
+  const list = []
+  for (const [key, value] of Object.entries(tags)) {
+    list.push(`${key}:${value}`.toLowerCase())
+  }
+  return list
+}
+
+function now () {
+  return Date.now() / 1e3
+}
+
+function mapToJsonArray (map) {
+  return Array.from(map.values()).map(v => v.toJSON())
+}
+
+class Metric {
+  constructor (namespace, metric, common, tags) {
+    this.namespace = namespace.toString()
+    this.metric = common ? metric : `nodejs.${metric}`
+    this.tags = tagArray(tags)
+    if (common) {
+      this.tags.push('lib_language:nodejs')
+      this.tags.push(`version:${process.version}`)
+    } else {
+      this.tags.push(`lib_version:${version}`)
+    }
+    this.common = common
+
+    this.points = []
+  }
+
+  toString () {
+    const { namespace, metric } = this
+    return `${namespace}.${metric}`
+  }
+
+  reset () {
+    this.points = []
+  }
+
+  track () {
+    throw new Error('not implemented')
+  }
+
+  toJSON () {
+    const { metric, points, interval, type, tags, common } = this
+    return {
+      metric,
+      points,
+      interval,
+      type,
+      tags,
+      common
+    }
+  }
+}
+
+class CountMetric extends Metric {
+  get type () {
+    return 'count'
+  }
+
+  inc (value) {
+    return this.track(value)
+  }
+
+  dec (value = -1) {
+    return this.track(value)
+  }
+
+  track (value = 1) {
+    if (this.points.length) {
+      this.points[0][1] += value
+    } else {
+      this.points.push([now(), value])
+    }
+  }
+}
+
+class DistributionMetric extends Metric {
+  get type () {
+    return 'distribution'
+  }
+
+  track (value = 1) {
+    this.points.push(value)
+  }
+
+  toJSON () {
+    const { metric, points, tags, common } = this
+    return {
+      metric,
+      points,
+      common,
+      tags
+    }
+  }
+}
+
+class GaugeMetric extends Metric {
+  get type () {
+    return 'gauge'
+  }
+
+  mark (value) {
+    return this.track(value)
+  }
+
+  track (value = 1) {
+    this.points.push([now(), value])
+  }
+}
+
+class RateMetric extends Metric {
+  constructor (namespace, metric, common, tags, interval) {
+    super(namespace, metric, common, tags)
+
+    this.interval = interval
+    this.rate = 0
+  }
+
+  get type () {
+    return 'rate'
+  }
+
+  reset () {
+    super.reset()
+    this.rate = 0
+  }
+
+  track (value = 1) {
+    this.rate += value
+    const rate = this.interval ? (this.rate / this.interval) : 0.0
+    this.points = [[now(), rate]]
+  }
+}
+
+const metricsTypes = {
+  count: CountMetric,
+  distribution: DistributionMetric,
+  gauge: GaugeMetric,
+  rate: RateMetric
+}
+
+class MetricsCollection extends Map {
+  constructor (namespace) {
+    super()
+    this.namespace = namespace
+  }
+
+  reset () {
+    for (const metric of this.values()) {
+      metric.reset()
+    }
+  }
+
+  toString () {
+    return this.namespace
+  }
+
+  toJSON () {
+    if (!this.size) return
+    const { namespace } = this
+    return {
+      namespace,
+      series: mapToJsonArray(this)
+    }
+  }
+}
+
+function getMetric (collection, type, name, tags, interval) {
+  const metricId = getId(type, collection, name, tags)
+
+  let metric = collection.get(metricId)
+  if (metric) return metric
+
+  const Factory = metricsTypes[type]
+  if (!Factory) {
+    throw new Error(`Unknown metric type ${type}`)
+  }
+
+  metric = new Factory(collection, name, true, tags, interval)
+  collection.set(metricId, metric)
+
+  return metric
+}
+
+class Namespace {
+  constructor (namespace) {
+    this.distributions = new MetricsCollection(namespace)
+    this.metrics = new MetricsCollection(namespace)
+  }
+
+  reset () {
+    this.metrics.reset()
+    this.distributions.reset()
+  }
+
+  count (name, tags) {
+    return getMetric(this.metrics, 'count', name, tags)
+  }
+
+  gauge (name, tags) {
+    return getMetric(this.metrics, 'gauge', name, tags)
+  }
+
+  rate (name, interval, tags) {
+    return getMetric(this.metrics, 'rate', name, tags, interval)
+  }
+
+  distribution (name, tags) {
+    return getMetric(this.distributions, 'distribution', name, tags)
+  }
+
+  toJSON () {
+    const { distributions, metrics } = this
+    return {
+      distributions: distributions.toJSON(),
+      metrics: metrics.toJSON()
+    }
+  }
+}
+
+class NamespaceManager extends Map {
+  namespace (name) {
+    let ns = this.get(name)
+    if (ns) return ns
+
+    ns = new Namespace(name)
+    this.set(name, ns)
+    return ns
+  }
+
+  toJSON () {
+    return mapToJsonArray(this)
+  }
+
+  send (config, application, host) {
+    for (const namespace of this.values()) {
+      const { metrics, distributions } = namespace.toJSON()
+
+      if (metrics) {
+        sendData(config, application, host, 'generate-metrics', metrics)
+      }
+
+      if (distributions) {
+        sendData(config, application, host, 'distributions', distributions)
+      }
+
+      // TODO: This could also be clear() but then it'd have to rebuild all
+      // metric instances on every send. This may be desirable if we want tags
+      // with high cardinality and variability over time.
+      namespace.reset()
+    }
+  }
+}
+
+const manager = new NamespaceManager()
+
+module.exports = {
+  CountMetric,
+  DistributionMetric,
+  GaugeMetric,
+  RateMetric,
+  MetricsCollection,
+  Namespace,
+  NamespaceManager,
+  manager
+}