dd-trace 2.4.2 → 2.7.0

package/packages/dd-trace/src/appsec/recommended.json

@@ -1,7 +1,7 @@
 {
   "version": "2.2",
   "metadata": {
-    "rules_version": "1.3.0"
+    "rules_version": "1.3.1"
   },
   "rules": [
     {
@@ -3040,7 +3040,9 @@
           "operator": "match_regex"
         }
       ],
-      "transformers": []
+      "transformers": [
+        "keys_only"
+      ]
     },
     {
       "id": "crs-942-360",
@@ -4097,15 +4099,23 @@
           "parameters": {
             "inputs": [
               {
-                "address": "server.request.headers.no_cookies"
+                "address": "server.request.query"
+              },
+              {
+                "address": "server.request.body"
+              },
+              {
+                "address": "server.request.path_params"
               }
             ],
-            "regex": "\\$(eq|ne|lte?|gte?|n?in)\\b"
+            "regex": "^\\$(eq|ne|(l|g)te?|n?in|not|(n|x|)or|and|regex|where|expr|exists)$"
           },
           "operator": "match_regex"
         }
       ],
-      "transformers": []
+      "transformers": [
+        "keys_only"
+      ]
     },
     {
       "id": "sqr-000-008",

package/packages/dd-trace/src/appsec/reporter.js

@@ -35,6 +35,8 @@ const RESPONSE_HEADERS_PASSLIST = [
   'content-type'
 ]
 
+const metricsQueue = new Map()
+
 function resolveHTTPRequest (context) {
   if (!context) return {}
 
@@ -82,6 +84,24 @@ function formatHeaderName (name) {
     .toLowerCase()
 }
 
+function reportMetrics (metrics, store) {
+  const req = store && store.get('req')
+  const topSpan = web.root(req)
+  if (!topSpan) return false
+
+  if (metrics.duration) {
+    topSpan.setTag('_dd.appsec.waf.duration', metrics.duration)
+  }
+
+  if (metrics.durationExt) {
+    topSpan.setTag('_dd.appsec.waf.duration_ext', metrics.durationExt)
+  }
+
+  if (metrics.rulesVersion) {
+    topSpan.setTag('_dd.appsec.event_rules.version', metrics.rulesVersion)
+  }
+}
+
 function reportAttack (attackData, store) {
   const req = store && store.get('req')
   const topSpan = web.root(req)
@@ -129,9 +149,17 @@ function reportAttack (attackData, store) {
   topSpan.addTags(newTags)
 }
 
-function finishAttacks (req, context) {
+function finishRequest (req, context) {
   const topSpan = web.root(req)
-  if (!topSpan || !context) return false
+  if (!topSpan) return false
+
+  if (metricsQueue.size) {
+    topSpan.addTags(Object.fromEntries(metricsQueue))
+
+    metricsQueue.clear()
+  }
+
+  if (!context || !topSpan.context()._tags['appsec.event']) return false
 
   const resolvedResponse = resolveHTTPResponse(context)
 
@@ -149,11 +177,13 @@ function setRateLimit (rateLimit) {
 }
 
 module.exports = {
+  metricsQueue,
   resolveHTTPRequest,
   resolveHTTPResponse,
   filterHeaders,
   formatHeaderName,
+  reportMetrics,
   reportAttack,
-  finishAttacks,
+  finishRequest,
   setRateLimit
 }

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -4,11 +4,11 @@ const callbacks = require('./callbacks')
 
 const appliedCallbacks = new Map()
 
-function applyRules (rules) {
+function applyRules (rules, config) {
   if (appliedCallbacks.has(rules)) return
 
   // for now there is only WAF
-  const callback = new callbacks.DDWAF(rules)
+  const callback = new callbacks.DDWAF(rules, config)
 
   appliedCallbacks.set(rules, callback)
 }

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const URL = require('url').URL
+const Writer = require('./writer')
+const Scheduler = require('../../../exporters/scheduler')
+
+class AgentlessCiVisibilityExporter {
+  constructor (config) {
+    const { flushInterval, tags, site, url } = config
+    this._url = url || new URL(`https://citestcycle-intake.${site}`)
+    this._writer = new Writer({ url: this._url, tags })
+
+    if (flushInterval > 0) {
+      this._scheduler = new Scheduler(() => this._writer.flush(), flushInterval)
+    }
+    this._scheduler && this._scheduler.start()
+  }
+
+  export (trace) {
+    this._writer.append(trace)
+
+    if (!this._scheduler) {
+      this._writer.flush()
+    }
+  }
+
+  flush () {
+    this._writer.flush()
+  }
+}
+
+module.exports = AgentlessCiVisibilityExporter

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -0,0 +1,51 @@
+'use strict'
+const request = require('../../../exporters/common/request')
+const log = require('../../../log')
+
+const { AgentlessCiVisibilityEncoder } = require('../../../encode/agentless-ci-visibility')
+const BaseWriter = require('../../../exporters/common/writer')
+
+class Writer extends BaseWriter {
+  constructor ({ url, tags }) {
+    super(...arguments)
+    const { 'runtime-id': runtimeId, env, service } = tags
+    this._url = url
+    this._encoder = new AgentlessCiVisibilityEncoder({ runtimeId, env, service })
+  }
+
+  _sendPayload (data, _, done) {
+    makeRequest(data, this._url, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the intake: ${res}`)
+      done()
+    })
+  }
+}
+
+function makeRequest (data, url, cb) {
+  const options = {
+    path: '/api/v2/citestcycle',
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/msgpack',
+      'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY
+    },
+    timeout: 15000
+  }
+
+  options.protocol = url.protocol
+  options.hostname = url.hostname
+  options.port = url.port
+
+  log.debug(() => `Request to the intake: ${JSON.stringify(options)}`)
+
+  request(data, options, false, (err, res) => {
+    cb(err, res)
+  })
+}
+
+module.exports = Writer

package/packages/dd-trace/src/config.js

@@ -66,6 +66,7 @@ class Config {
       process.env.DD_TRACE_URL,
       null
     )
+    const DD_CIVISIBILITY_AGENTLESS_URL = process.env.DD_CIVISIBILITY_AGENTLESS_URL
     const DD_SERVICE = options.service ||
       process.env.DD_SERVICE ||
       process.env.DD_SERVICE_NAME ||
@@ -90,6 +91,10 @@ class Config {
       process.env.DD_TRACE_STARTUP_LOGS,
       false
     )
+    const DD_TRACE_TELEMETRY_ENABLED = coalesce(
+      process.env.DD_TRACE_TELEMETRY_ENABLED,
+      true
+    )
     const DD_TRACE_DEBUG = coalesce(
       process.env.DD_TRACE_DEBUG,
       false
@@ -145,10 +150,29 @@ class Config {
       path.join(__dirname, 'appsec', 'recommended.json')
     )
     const DD_APPSEC_TRACE_RATE_LIMIT = coalesce(
-      appsec.rateLimit,
-      process.env.DD_APPSEC_TRACE_RATE_LIMIT,
+      parseInt(appsec.rateLimit),
+      parseInt(process.env.DD_APPSEC_TRACE_RATE_LIMIT),
       100
     )
+    const DD_APPSEC_WAF_TIMEOUT = coalesce(
+      parseInt(appsec.wafTimeout),
+      parseInt(process.env.DD_APPSEC_WAF_TIMEOUT),
+      5e3 // µs
+    )
+    const DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP = coalesce(
+      appsec.obfuscatorKeyRegex,
+      process.env.DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
+      `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?)key)|token|consumer_?(?:id|key|se\
+cret)|sign(?:ed|ature)|bearer|authorization`
+    )
+    const DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP = coalesce(
+      appsec.obfuscatorValueRegex,
+      process.env.DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP,
+      `(?i)(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|public_?|access_?|secret_?)key(?:_?id)?|to\
+ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)(?:\\s*=[^;]|"\\s*:\\s*"[^"]+")|bearer\
+\\s+[a-z0-9\\._\\-]+|token:[a-z0-9]{13}|gh[opsu]_[0-9a-zA-Z]{36}|ey[I-L][\\w=-]+\\.ey[I-L][\\w=-]+(?:\\.[\\w.+\\/=-]+)?\
+|[\\-]{5}BEGIN[a-z\\s]+PRIVATE\\sKEY[\\-]{5}[^\\-]+[\\-]{5}END[a-z\\s]+PRIVATE\\sKEY|ssh-rsa\\s*[a-z0-9\\/\\.+]{100,}`
+    )
 
     const sampler = (options.experimental && options.experimental.sampler) || {}
     const ingestion = options.ingestion || {}
@@ -171,7 +195,8 @@ class Config {
     this.debug = isTrue(DD_TRACE_DEBUG)
     this.logInjection = isTrue(DD_LOGS_INJECTION)
     this.env = DD_ENV
-    this.url = getAgentUrl(DD_TRACE_AGENT_URL, options)
+    this.url = DD_CIVISIBILITY_AGENTLESS_URL ? new URL(DD_CIVISIBILITY_AGENTLESS_URL)
+      : getAgentUrl(DD_TRACE_AGENT_URL, options)
     this.site = coalesce(options.site, process.env.DD_SITE, 'datadoghq.com')
     this.hostname = DD_AGENT_HOST || (this.url && this.url.hostname)
     this.port = String(DD_TRACE_AGENT_PORT || (this.url && this.url.port))
@@ -212,11 +237,15 @@ class Config {
     }
     this.lookup = options.lookup
     this.startupLogs = isTrue(DD_TRACE_STARTUP_LOGS)
+    this.telemetryEnabled = isTrue(DD_TRACE_TELEMETRY_ENABLED)
     this.protocolVersion = DD_TRACE_AGENT_PROTOCOL_VERSION
     this.appsec = {
       enabled: isTrue(DD_APPSEC_ENABLED),
       rules: DD_APPSEC_RULES,
-      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT
+      rateLimit: DD_APPSEC_TRACE_RATE_LIMIT,
+      wafTimeout: DD_APPSEC_WAF_TIMEOUT,
+      obfuscatorKeyRegex: DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP,
+      obfuscatorValueRegex: DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     }
 
     tagger.add(this.tags, {

package/packages/dd-trace/src/encode/0.4.js

@@ -196,7 +196,6 @@ class AgentEncoder {
 
     for (const key of keys) {
       if (typeof value[key] !== 'string' && typeof value[key] !== 'number') return
-
       length++
 
       this._encodeString(bytes, key)

package/packages/dd-trace/src/encode/agentless-ci-visibility.js

@@ -0,0 +1,193 @@
+'use strict'
+const { truncateSpan, normalizeSpan } = require('./tags-processors')
+const Chunk = require('./chunk')
+const { AgentEncoder } = require('./0.4')
+
+const ENCODING_VERSION = 1
+
+function formatSpan (span) {
+  return {
+    type: span.type === 'test' ? 'test' : 'span',
+    version: ENCODING_VERSION,
+    content: normalizeSpan(truncateSpan(span))
+  }
+}
+
+class AgentlessCiVisibilityEncoder extends AgentEncoder {
+  constructor ({ runtimeId, service, env }) {
+    super(...arguments)
+    this._events = []
+    this.runtimeId = runtimeId
+    this.service = service
+    this.env = env
+    this._traceBytes = new Chunk()
+    this._stringBytes = new Chunk()
+    this._stringCount = 0
+    this._stringMap = {}
+
+    // Used to keep track of the number of encoded events to update the
+    // length of `payload.events` when calling `makePayload`
+    this._eventCount = 0
+
+    this.reset()
+  }
+
+  _encodeEventContent (bytes, content) {
+    this._encodeMapPrefix(bytes, content)
+    if (content.type) {
+      this._encodeString(bytes, 'type')
+      this._encodeString(bytes, content.type)
+    }
+    this._encodeString(bytes, 'trace_id')
+    this._encodeId(bytes, content.trace_id)
+    this._encodeString(bytes, 'span_id')
+    this._encodeId(bytes, content.span_id)
+    this._encodeString(bytes, 'parent_id')
+    this._encodeId(bytes, content.parent_id)
+    this._encodeString(bytes, 'name')
+    this._encodeString(bytes, content.name)
+    this._encodeString(bytes, 'resource')
+    this._encodeString(bytes, content.resource)
+    this._encodeString(bytes, 'service')
+    this._encodeString(bytes, content.service)
+    this._encodeString(bytes, 'error')
+    this._encodeNumber(bytes, content.error)
+    this._encodeString(bytes, 'start')
+    this._encodeNumber(bytes, content.start)
+    this._encodeString(bytes, 'duration')
+    this._encodeNumber(bytes, content.duration)
+    this._encodeString(bytes, 'meta')
+    this._encodeMap(bytes, content.meta)
+    this._encodeString(bytes, 'metrics')
+    this._encodeMap(bytes, content.metrics)
+  }
+
+  _encodeEvent (bytes, event) {
+    this._encodeMapPrefix(bytes, event)
+    this._encodeString(bytes, 'type')
+    this._encodeString(bytes, event.type)
+
+    this._encodeString(bytes, 'version')
+    this._encodeNumber(bytes, event.version)
+
+    this._encodeString(bytes, 'content')
+    this._encodeEventContent(bytes, event.content)
+  }
+
+  _encodeNumber (bytes, value) {
+    if (Math.floor(value) !== value) { // float 64
+      return this._encodeFloat(bytes, value)
+    }
+    return this._encodeLong(bytes, value)
+  }
+
+  _encodeLong (bytes, value) {
+    const isPositive = value >= 0
+
+    const hi = isPositive ? (value / Math.pow(2, 32)) >> 0 : Math.floor(value / Math.pow(2, 32))
+    const lo = value >>> 0
+    const flag = isPositive ? 0xcf : 0xd3
+
+    const buffer = bytes.buffer
+    const offset = bytes.length
+
+    // int 64
+    bytes.reserve(9)
+    bytes.length += 9
+
+    buffer[offset] = flag
+    buffer[offset + 1] = hi >> 24
+    buffer[offset + 2] = hi >> 16
+    buffer[offset + 3] = hi >> 8
+    buffer[offset + 4] = hi
+    buffer[offset + 5] = lo >> 24
+    buffer[offset + 6] = lo >> 16
+    buffer[offset + 7] = lo >> 8
+    buffer[offset + 8] = lo
+  }
+
+  _encodeMapPrefix (bytes, map) {
+    const keys = Object.keys(map)
+    const buffer = bytes.buffer
+    const offset = bytes.length
+
+    bytes.reserve(5)
+    bytes.length += 5
+    buffer[offset] = 0xdf
+    buffer[offset + 1] = keys.length >> 24
+    buffer[offset + 2] = keys.length >> 16
+    buffer[offset + 3] = keys.length >> 8
+    buffer[offset + 4] = keys.length
+  }
+
+  _encode (bytes, trace) {
+    this._eventCount += trace.length
+    const events = trace.map(formatSpan)
+
+    for (const event of events) {
+      this._encodeEvent(bytes, event)
+    }
+  }
+
+  makePayload () {
+    const bytes = this._traceBytes
+    const eventsOffset = this._eventsOffset
+    const eventsCount = this._eventCount
+
+    bytes.buffer[eventsOffset] = 0xdd
+    bytes.buffer[eventsOffset + 1] = eventsCount >> 24
+    bytes.buffer[eventsOffset + 2] = eventsCount >> 16
+    bytes.buffer[eventsOffset + 3] = eventsCount >> 8
+    bytes.buffer[eventsOffset + 4] = eventsCount
+
+    const traceSize = bytes.length
+    const buffer = Buffer.allocUnsafe(traceSize)
+
+    bytes.buffer.copy(buffer, 0, 0, bytes.length)
+
+    this.reset()
+
+    return buffer
+  }
+
+  _encodePayloadStart (bytes) {
+    // encodes the payload up to `events`. `events` will be encoded via _encode
+    const payload = {
+      version: ENCODING_VERSION,
+      metadata: {
+        '*': {
+          'language': 'javascript'
+        }
+      },
+      events: []
+    }
+
+    if (this.env) {
+      payload.metadata['*'].env = this.env
+    }
+    if (this.runtimeId) {
+      payload.metadata['*']['runtime-id'] = this.runtimeId
+    }
+
+    this._encodeMapPrefix(bytes, payload)
+    this._encodeString(bytes, 'version')
+    this._encodeNumber(bytes, payload.version)
+    this._encodeString(bytes, 'metadata')
+    this._encodeMapPrefix(bytes, payload.metadata)
+    this._encodeString(bytes, '*')
+    this._encodeMap(bytes, payload.metadata['*'])
+    this._encodeString(bytes, 'events')
+    // Get offset of the events list to update the length of the array when calling `makePayload`
+    this._eventsOffset = bytes.length
+    bytes.reserve(5)
+    bytes.length += 5
+  }
+
+  reset () {
+    this._reset()
+    this._eventCount = 0
+    this._encodePayloadStart(this._traceBytes)
+  }
+}
+
+module.exports = { AgentlessCiVisibilityEncoder }

package/packages/dd-trace/src/encode/tags-processors.js

@@ -0,0 +1,116 @@
+// From agent truncators: https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/agent/truncator.go
+
+// Values from: https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/truncate.go#L22-L27
+// MAX_RESOURCE_NAME_LENGTH the maximum length a span resource can have
+const MAX_RESOURCE_NAME_LENGTH = 5000
+// MAX_META_KEY_LENGTH the maximum length of metadata key
+const MAX_META_KEY_LENGTH = 200
+// MAX_META_VALUE_LENGTH the maximum length of metadata value
+const MAX_META_VALUE_LENGTH = 25000
+// MAX_METRIC_KEY_LENGTH the maximum length of a metric name key
+const MAX_METRIC_KEY_LENGTH = MAX_META_KEY_LENGTH
+// MAX_METRIC_VALUE_LENGTH the maximum length of a metric name value
+const MAX_METRIC_VALUE_LENGTH = MAX_META_VALUE_LENGTH
+
+// From agent normalizer:
+// https://github.com/DataDog/datadog-agent/blob/main/pkg/trace/traceutil/normalize.go
+// DEFAULT_SPAN_NAME is the default name we assign a span if it's missing and we have no reasonable fallback
+const DEFAULT_SPAN_NAME = 'unnamed_operation'
+// DEFAULT_SERVICE_NAME is the default name we assign a service if it's missing and we have no reasonable fallback
+const DEFAULT_SERVICE_NAME = 'unnamed-service'
+// MAX_NAME_LENGTH the maximum length a name can have
+const MAX_NAME_LENGTH = 100
+// MAX_SERVICE_LENGTH the maximum length a service can have
+const MAX_SERVICE_LENGTH = 100
+// MAX_TYPE_LENGTH the maximum length a span type can have
+const MAX_TYPE_LENGTH = 100
+
+const fromEntries = Object.fromEntries || (entries =>
+  entries.reduce((obj, [k, v]) => Object.assign(obj, { [k]: v }), {}))
+
+function truncateToLength (value, maxLength) {
+  if (!value) {
+    return value
+  }
+  if (value.length > maxLength) {
+    return `${value.slice(0, maxLength)}...`
+  }
+  return value
+}
+
+function truncateSpan (span) {
+  return fromEntries(Object.entries(span).map(([key, value]) => {
+    switch (key) {
+      case 'resource':
+        return ['resource', truncateToLength(value, MAX_RESOURCE_NAME_LENGTH)]
+      case 'meta':
+        return ['meta', fromEntries(Object.entries(value).map(([metaKey, metaValue]) =>
+          [truncateToLength(metaKey, MAX_META_KEY_LENGTH), truncateToLength(metaValue, MAX_META_VALUE_LENGTH)]
+        ))]
+      case 'metrics':
+        return ['metrics', fromEntries(Object.entries(value).map(([metricsKey, metricsValue]) =>
+          [truncateToLength(metricsKey, MAX_METRIC_KEY_LENGTH), truncateToLength(metricsValue, MAX_METRIC_VALUE_LENGTH)]
+        ))]
+      default:
+        return [key, value]
+    }
+  }))
+}
+
+function normalizeSpan (span) {
+  const normalizedSpan = fromEntries(Object.entries(span).map(([key, value]) => {
+    switch (key) {
+      case 'service':
+        if (!value) {
+          return [key, DEFAULT_SERVICE_NAME]
+        }
+        if (value.length > MAX_SERVICE_LENGTH) {
+          return [key, value.slice(0, MAX_SERVICE_LENGTH)]
+        }
+        break
+      case 'name':
+        if (!value) {
+          return [key, DEFAULT_SPAN_NAME]
+        }
+        if (value.length > MAX_NAME_LENGTH) {
+          return [key, value.slice(0, MAX_NAME_LENGTH)]
+        }
+        break
+      case 'resource':
+        if (!value) {
+          return [key, span.name || DEFAULT_SPAN_NAME]
+        }
+        break
+      case 'type':
+        if (!value) {
+          return [key, value]
+        }
+        if (value.length > MAX_TYPE_LENGTH) {
+          return [key, value.slice(0, MAX_TYPE_LENGTH)]
+        }
+    }
+    return [key, value]
+  }))
+  if (!normalizedSpan.service) {
+    normalizedSpan.service = DEFAULT_SERVICE_NAME
+  }
+  if (!normalizedSpan.name) {
+    normalizedSpan.name = DEFAULT_SPAN_NAME
+  }
+  return normalizedSpan
+}
+
+module.exports = {
+  truncateSpan,
+  normalizeSpan,
+  MAX_META_KEY_LENGTH,
+  MAX_META_VALUE_LENGTH,
+  MAX_METRIC_KEY_LENGTH,
+  MAX_METRIC_VALUE_LENGTH,
+  MAX_NAME_LENGTH,
+  MAX_SERVICE_LENGTH,
+  MAX_TYPE_LENGTH,
+  MAX_RESOURCE_NAME_LENGTH,
+  DEFAULT_SPAN_NAME,
+  DEFAULT_SERVICE_NAME
+}

package/packages/dd-trace/src/exporter.js

@@ -2,6 +2,7 @@
 
 const AgentExporter = require('./exporters/agent')
 const LogExporter = require('./exporters/log')
+const AgentlessCiVisibilityExporter = require('./ci-visibility/exporters/agentless')
 const exporters = require('../../../ext/exporters')
 const fs = require('fs')
 const constants = require('./constants')
@@ -15,6 +16,8 @@ module.exports = name => {
       return LogExporter
     case exporters.AGENT:
       return AgentExporter
+    case exporters.DATADOG:
+      return AgentlessCiVisibilityExporter
     default:
       return inAWSLambda && !usingLambdaExtension ? LogExporter : AgentExporter
   }

package/packages/dd-trace/src/exporters/agent/index.js

@@ -3,7 +3,7 @@
 const URL = require('url').URL
 const log = require('../../log')
 const Writer = require('./writer')
-const Scheduler = require('./scheduler')
+const Scheduler = require('../scheduler')
 
 class AgentExporter {
   constructor ({ url, hostname, port, flushInterval, lookup, protocolVersion }, prioritySampler) {