dd-trace 2.22.2 → 2.23.0

package/packages/datadog-plugin-mocha/src/index.js

@@ -14,7 +14,10 @@ const {
   getTestSuiteCommonTags,
   TEST_SUITE_ID,
   TEST_SESSION_ID,
-  TEST_COMMAND
+  TEST_COMMAND,
+  TEST_ITR_TESTS_SKIPPED,
+  TEST_SESSION_ITR_CODE_COVERAGE_ENABLED,
+  TEST_SESSION_ITR_SKIPPING_ENABLED
 } = require('../../dd-trace/src/plugins/util/test')
 const { COMPONENT } = require('../../dd-trace/src/constants')
 
@@ -31,9 +34,6 @@ class MochaPlugin extends CiPlugin {
     this.sourceRoot = process.cwd()
 
     this.addSub('ci:mocha:test-suite:code-coverage', ({ coverageFiles, suiteFile }) => {
-      if (!this.config.isAgentlessEnabled || !this.config.isIntelligentTestRunnerEnabled) {
-        return
-      }
       if (!this.itrConfig || !this.itrConfig.isCodeCoverageEnabled) {
         return
       }
@@ -49,9 +49,6 @@ class MochaPlugin extends CiPlugin {
     })
 
     this.addSub('ci:mocha:session:start', (command) => {
-      if (!this.config.isAgentlessEnabled) {
-        return
-      }
       const childOf = getTestParentSpan(this.tracer)
       const testSessionSpanMetadata = getTestSessionCommonTags(command, this.tracer._version)
 
@@ -67,9 +64,6 @@ class MochaPlugin extends CiPlugin {
     })
 
     this.addSub('ci:mocha:test-suite:start', (suite) => {
-      if (!this.config.isAgentlessEnabled) {
-        return
-      }
       const store = storage.getStore()
       const testSuiteMetadata = getTestSuiteCommonTags(
         this.command,
@@ -89,9 +83,6 @@ class MochaPlugin extends CiPlugin {
     })
 
     this.addSub('ci:mocha:test-suite:finish', (status) => {
-      if (!this.config.isAgentlessEnabled) {
-        return
-      }
       const span = storage.getStore().span
       // the test status of the suite may have been set in ci:mocha:test-suite:error already
       if (!span.context()._tags[TEST_STATUS]) {
@@ -101,9 +92,6 @@ class MochaPlugin extends CiPlugin {
     })
 
     this.addSub('ci:mocha:test-suite:error', (err) => {
-      if (!this.config.isAgentlessEnabled) {
-        return
-      }
       const span = storage.getStore().span
       span.setTag('error', err)
       span.setTag(TEST_STATUS, 'fail')
@@ -151,14 +139,19 @@ class MochaPlugin extends CiPlugin {
       this._testNameToParams[name] = params
     })
 
-    this.addSub('ci:mocha:session:finish', (status) => {
+    this.addSub('ci:mocha:session:finish', ({ status, isSuitesSkipped }) => {
       if (this.testSessionSpan) {
+        const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.itrConfig || {}
         this.testSessionSpan.setTag(TEST_STATUS, status)
+        this.testSessionSpan.setTag(TEST_ITR_TESTS_SKIPPED, isSuitesSkipped ? 'true' : 'false')
+        this.testSessionSpan.setTag(TEST_SESSION_ITR_SKIPPING_ENABLED, isSuitesSkippingEnabled ? 'true' : 'false')
+        this.testSessionSpan.setTag(TEST_SESSION_ITR_CODE_COVERAGE_ENABLED, isCodeCoverageEnabled ? 'true' : 'false')
+
         this.testSessionSpan.finish()
         finishAllTraceSpans(this.testSessionSpan)
       }
-      this.tracer._exporter._writer.flush()
       this.itrConfig = null
+      this.tracer._exporter.flush()
     })
   }
 

package/packages/datadog-plugin-mongodb-core/src/index.js

@@ -8,7 +8,7 @@ class MongodbCorePlugin extends DatabasePlugin {
 
   start ({ ns, ops, options = {}, name }) {
     const query = getQuery(ops)
-    const resource = truncate(getResource(ns, query, name))
+    const resource = truncate(getResource(this, ns, query, name))
 
     this.startSpan('mongodb.query', {
       service: this.config.service,
@@ -31,10 +31,10 @@ function getQuery (cmd) {
   if (cmd.filter) return JSON.stringify(limitDepth(cmd.filter))
 }
 
-function getResource (ns, query, operationName) {
+function getResource (plugin, ns, query, operationName) {
   const parts = [operationName, ns]
 
-  if (query) {
+  if (plugin.config.queryInResourceName && query) {
     parts.push(query)
   }
 

package/packages/dd-trace/src/appsec/callbacks/ddwaf.js

@@ -123,6 +123,10 @@ class WAFCallback {
     return result.actions
   }
 
+  updateRuleData (ruleData) {
+    this.ddwaf.updateRuleData(ruleData)
+  }
+
   clear () {
     this.ddwaf.dispose()
 

package/packages/dd-trace/src/appsec/index.js

@@ -1,8 +1,10 @@
 'use strict'
 
 const fs = require('fs')
+const path = require('path')
 const log = require('../log')
 const RuleManager = require('./rule_manager')
+const remoteConfig = require('./remote_config')
 const { incomingHttpRequestStart, incomingHttpRequestEnd } = require('./gateway/channels')
 const Gateway = require('./gateway/engine')
 const addresses = require('./addresses')
@@ -17,16 +19,18 @@ function enable (config) {
   try {
     // TODO: enable dc_blocking: config.appsec.blocking === true
 
-    let rules = fs.readFileSync(config.appsec.rules)
+    let rules = fs.readFileSync(config.appsec.rules || path.join(__dirname, 'recommended.json'))
     rules = JSON.parse(rules)
 
     RuleManager.applyRules(rules, config.appsec)
+    remoteConfig.enableAsmData(config.appsec)
   } catch (err) {
     log.error('Unable to start AppSec')
     log.error(err)
 
     // abort AppSec start
     RuleManager.clearAllRules()
+    remoteConfig.disableAsmData()
     return
   }
 
@@ -116,6 +120,7 @@ function disable () {
   isEnabled = false
 
   RuleManager.clearAllRules()
+  remoteConfig.disableAsmData()
 
   // Channel#unsubscribe() is undefined for non active channels
   if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(incomingHttpStartTranslator)

package/packages/dd-trace/src/appsec/remote_config/index.js

@@ -2,9 +2,12 @@
 
 const RemoteConfigManager = require('./manager')
 const RemoteConfigCapabilities = require('./capabilities')
+const RuleManager = require('../rule_manager')
+
+let rc
 
 function enable (config) {
-  const rc = new RemoteConfigManager(config)
+  rc = new RemoteConfigManager(config)
 
   if (config.appsec.enabled === undefined) { // only activate ASM_FEATURES when conf is not set locally
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_ACTIVATION, true)
@@ -29,6 +32,24 @@ function enable (config) {
   }
 }
 
+function enableAsmData (appsecConfig) {
+  if (rc && appsecConfig && appsecConfig.rules === undefined) {
+    rc.on('ASM_DATA', _asmDataListener)
+  }
+}
+
+function disableAsmData () {
+  if (rc) {
+    rc.off('ASM_DATA', _asmDataListener)
+  }
+}
+
+function _asmDataListener (action, ruleData, ruleId) {
+  RuleManager.updateAsmData(action, ruleData, ruleId)
+}
+
 module.exports = {
-  enable
+  enable,
+  enableAsmData,
+  disableAsmData
 }

package/packages/dd-trace/src/appsec/remote_config/manager.js

@@ -223,7 +223,7 @@ class RemoteConfigManager extends EventEmitter {
     for (const item of list) {
       try {
         // TODO: do we want to pass old and new config ?
-        this.emit(item.product, action, item.file)
+        this.emit(item.product, action, item.file, item.id)
 
         item.apply_state = 2
       } catch (err) {

package/packages/dd-trace/src/appsec/rule_manager.js

@@ -4,6 +4,7 @@ const callbacks = require('./callbacks')
 const Gateway = require('./gateway/engine')
 
 const appliedCallbacks = new Map()
+const appliedAsmData = new Map()
 
 function applyRules (rules, config) {
   if (appliedCallbacks.has(rules)) return
@@ -14,6 +15,60 @@ function applyRules (rules, config) {
   appliedCallbacks.set(rules, callback)
 }
 
+function updateAsmData (action, asmData, asmDataId) {
+  if (action === 'unapply') {
+    appliedAsmData.delete(asmDataId)
+  } else {
+    appliedAsmData.set(asmDataId, asmData)
+  }
+
+  const mergedRuleData = mergeRuleData(appliedAsmData.values())
+  for (const callback of appliedCallbacks.values()) {
+    callback.updateRuleData(mergedRuleData)
+  }
+}
+
+function mergeRuleData (asmDataValues) {
+  const mergedRulesData = new Map()
+  for (const asmData of asmDataValues) {
+    if (!asmData.rules_data) continue
+    for (const rulesData of asmData.rules_data) {
+      const key = `${rulesData.id}+${rulesData.type}`
+      if (mergedRulesData.has(key)) {
+        const existingRulesData = mergedRulesData.get(key)
+        rulesData.data.reduce(rulesReducer, existingRulesData.data)
+      } else {
+        mergedRulesData.set(key, copyRulesData(rulesData))
+      }
+    }
+  }
+  return [...mergedRulesData.values()]
+}
+
+function rulesReducer (existingEntries, rulesDataEntry) {
+  const existingEntry = existingEntries.find((entry) => entry.value === rulesDataEntry.value)
+  if (existingEntry && !('expiration' in existingEntry)) return existingEntries
+  if (existingEntry && 'expiration' in rulesDataEntry && rulesDataEntry.expiration > existingEntry.expiration) {
+    existingEntry.expiration = rulesDataEntry.expiration
+  } else if (existingEntry && !('expiration' in rulesDataEntry)) {
+    delete existingEntry.expiration
+  } else if (!existingEntry) {
+    existingEntries.push({ ...rulesDataEntry })
+  }
+  return existingEntries
+}
+
+function copyRulesData (rulesData) {
+  const copy = { ...rulesData }
+  if (copy.data) {
+    const data = []
+    copy.data.forEach(item => {
+      data.push({ ...item })
+    })
+    copy.data = data
+  }
+  return copy
+}
 function clearAllRules () {
   Gateway.manager.clear()
 
@@ -22,9 +77,11 @@ function clearAllRules () {
 
     appliedCallbacks.delete(key)
   }
+  appliedAsmData.clear()
 }
 
 module.exports = {
   applyRules,
-  clearAllRules
+  clearAllRules,
+  updateAsmData
 }

package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js

@@ -0,0 +1,62 @@
+'use strict'
+
+const AgentWriter = require('../../../exporters/agent/writer')
+const AgentlessWriter = require('../agentless/writer')
+const CoverageWriter = require('../agentless/coverage-writer')
+const CiVisibilityExporter = require('../ci-visibility-exporter')
+
+const AGENT_EVP_PROXY_PATH = '/evp_proxy/v2'
+
+function getIsEvpCompatible (err, agentInfo) {
+  return !err && agentInfo.endpoints.some(url => url.includes(AGENT_EVP_PROXY_PATH))
+}
+
+class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
+  constructor (config) {
+    super(config)
+
+    const {
+      tags,
+      prioritySampler,
+      lookup,
+      protocolVersion,
+      headers,
+      isGitUploadEnabled
+    } = config
+
+    this.getAgentInfo((err, agentInfo) => {
+      this._isInitialized = true
+      const isEvpCompatible = getIsEvpCompatible(err, agentInfo)
+      if (isEvpCompatible) {
+        this._isUsingEvpProxy = true
+        this._writer = new AgentlessWriter({
+          url: this._url,
+          tags,
+          evpProxyPrefix: AGENT_EVP_PROXY_PATH
+        })
+        this._coverageWriter = new CoverageWriter({
+          url: this._url,
+          evpProxyPrefix: AGENT_EVP_PROXY_PATH
+        })
+        if (isGitUploadEnabled) {
+          this.sendGitMetadata({ url: this._url, isEvpProxy: true })
+        }
+      } else {
+        this._writer = new AgentWriter({
+          url: this._url,
+          prioritySampler,
+          lookup,
+          protocolVersion,
+          headers
+        })
+        // coverages will never be used, so we discard them
+        this._coverageBuffer = []
+      }
+      this._resolveCanUseCiVisProtocol(isEvpCompatible)
+      this.exportUncodedTraces()
+      this.exportUncodedCoverages()
+    })
+  }
+}
+
+module.exports = AgentProxyCiVisibilityExporter

package/packages/dd-trace/src/ci-visibility/exporters/agentless/coverage-writer.js

@@ -12,10 +12,11 @@ function safeJSONStringify (value) {
 }
 
 class Writer extends BaseWriter {
-  constructor ({ url }) {
+  constructor ({ url, evpProxyPrefix = '' }) {
     super(...arguments)
     this._url = url
     this._encoder = new CoverageCIVisibilityEncoder(this)
+    this._evpProxyPrefix = evpProxyPrefix
   }
 
   _sendPayload (form, _, done) {
@@ -30,6 +31,12 @@ class Writer extends BaseWriter {
       url: this._url
     }
 
+    if (this._evpProxyPrefix) {
+      options.path = `${this._evpProxyPrefix}/api/v2/citestcov`
+      delete options.headers['dd-api-key']
+      options.headers['X-Datadog-EVP-Subdomain'] = 'event-platform-intake'
+    }
+
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
 
     request(form, options, (err, res) => {

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -3,73 +3,25 @@
 const URL = require('url').URL
 const Writer = require('./writer')
 const CoverageWriter = require('./coverage-writer')
+const CiVisibilityExporter = require('../ci-visibility-exporter')
 
-const log = require('../../../log')
-
-class AgentlessCiVisibilityExporter {
+class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
   constructor (config) {
-    this._config = config
-    const { tags, site, url, isIntelligentTestRunnerEnabled } = config
-    this._isIntelligentTestRunnerEnabled = isIntelligentTestRunnerEnabled
+    super(config)
+    const { tags, site, url, isGitUploadEnabled } = config
+    // we don't need to request /info because we are using agentless by configuration
+    this._isInitialized = true
+    this._resolveCanUseCiVisProtocol(true)
+
     this._url = url || new URL(`https://citestcycle-intake.${site}`)
     this._writer = new Writer({ url: this._url, tags })
-    this._timer = undefined
-    this._coverageTimer = undefined
 
     this._coverageUrl = url || new URL(`https://event-platform-intake.${site}`)
     this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
 
-    process.once('beforeExit', () => {
-      this._writer.flush()
-      this._coverageWriter.flush()
-    })
-  }
-
-  exportCoverage ({ span, coverageFiles }) {
-    const formattedCoverage = {
-      traceId: span.context()._traceId,
-      spanId: span.context()._spanId,
-      files: coverageFiles
-    }
-    this._coverageWriter.append(formattedCoverage)
-
-    const { flushInterval } = this._config
-
-    if (flushInterval === 0) {
-      this._coverageWriter.flush()
-    } else if (flushInterval > 0 && !this._coverageTimer) {
-      this._coverageTimer = setTimeout(() => {
-        this._coverageWriter.flush()
-        this._coverageTimer = clearTimeout(this._coverageTimer)
-      }, flushInterval).unref()
-    }
-  }
-
-  export (trace) {
-    this._writer.append(trace)
-
-    const { flushInterval } = this._config
-
-    if (flushInterval === 0) {
-      this._writer.flush()
-    } else if (flushInterval > 0 && !this._timer) {
-      this._timer = setTimeout(() => {
-        this._writer.flush()
-        this._timer = clearTimeout(this._timer)
-      }, flushInterval).unref()
-    }
-  }
-
-  setUrl (url, coverageUrl = url) {
-    try {
-      url = new URL(url)
-      coverageUrl = new URL(coverageUrl)
-      this._url = url
-      this._coverageUrl = coverageUrl
-      this._writer.setUrl(url)
-      this._coverageWriter.setUrl(coverageUrl)
-    } catch (e) {
-      log.error(e)
+    if (isGitUploadEnabled) {
+      const gitUrl = url || new URL(`https://api.${site}`)
+      this.sendGitMetadata({ url: gitUrl })
     }
   }
 }

package/packages/dd-trace/src/ci-visibility/exporters/agentless/writer.js

@@ -12,11 +12,12 @@ function safeJSONStringify (value) {
 }
 
 class Writer extends BaseWriter {
-  constructor ({ url, tags }) {
+  constructor ({ url, tags, evpProxyPrefix = '' }) {
     super(...arguments)
     const { 'runtime-id': runtimeId, env, service } = tags
     this._url = url
     this._encoder = new AgentlessCiVisibilityEncoder(this, { runtimeId, env, service })
+    this._evpProxyPrefix = evpProxyPrefix
   }
 
   _sendPayload (data, _, done) {
@@ -31,7 +32,14 @@ class Writer extends BaseWriter {
       url: this._url
     }
 
+    if (this._evpProxyPrefix) {
+      options.path = `${this._evpProxyPrefix}/api/v2/citestcycle`
+      delete options.headers['dd-api-key']
+      options.headers['X-Datadog-EVP-Subdomain'] = 'citestcycle-intake'
+    }
+
     log.debug(() => `Request to the intake: ${safeJSONStringify(options)}`)
+
     request(data, options, (err, res) => {
       if (err) {
         log.error(err)

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -0,0 +1,196 @@
+'use strict'
+
+const URL = require('url').URL
+
+const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata')
+const { getItrConfiguration: getItrConfigurationRequest } = require('../intelligent-test-runner/get-itr-configuration')
+const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligent-test-runner/get-skippable-suites')
+const log = require('../../log')
+const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
+
+function getIsTestSessionTrace (trace) {
+  return trace.some(span =>
+    span.type === 'test_session_end' || span.type === 'test_suite_end'
+  )
+}
+
+class CiVisibilityExporter extends AgentInfoExporter {
+  constructor (config) {
+    super(config)
+    this._timer = undefined
+    this._coverageTimer = undefined
+    this._coverageBuffer = []
+    // The library can use new features like ITR and test suite level visibility
+    // AKA CI Vis Protocol
+    this._canUseCiVisProtocol = false
+
+    // TODO: add timeout to reject this promise
+    this._gitUploadPromise = new Promise(resolve => {
+      this._resolveGit = resolve
+    })
+
+    // TODO: add timeout to reject this promise
+    this._canUseCiVisProtocolPromise = new Promise(resolve => {
+      this._resolveCanUseCiVisProtocol = (canUseCiVisProtocol) => {
+        this._canUseCiVisProtocol = canUseCiVisProtocol
+        resolve(canUseCiVisProtocol)
+      }
+    })
+
+    process.once('beforeExit', () => {
+      if (this._writer) {
+        this._writer.flush()
+      }
+      if (this._coverageWriter) {
+        this._coverageWriter.flush()
+      }
+    })
+  }
+
+  shouldRequestSkippableSuites () {
+    return !!(this._config.isIntelligentTestRunnerEnabled &&
+      this._canUseCiVisProtocol &&
+      this._itrConfig &&
+      this._itrConfig.isSuitesSkippingEnabled)
+  }
+
+  shouldRequestItrConfiguration () {
+    return this._config.isIntelligentTestRunnerEnabled
+  }
+
+  canReportSessionTraces () {
+    return this._canUseCiVisProtocol
+  }
+
+  canReportCodeCoverage () {
+    return this._canUseCiVisProtocol &&
+      this._itrConfig &&
+      this._itrConfig.isCodeCoverageEnabled
+  }
+
+  // We can't call the skippable endpoint until git upload has finished,
+  // hence the this._gitUploadPromise.then
+  getSkippableSuites (testConfiguration, callback) {
+    if (!this.shouldRequestSkippableSuites()) {
+      return callback(null, [])
+    }
+    this._gitUploadPromise.then(gitUploadError => {
+      if (gitUploadError) {
+        return callback(gitUploadError, [])
+      }
+      const configuration = {
+        url: this._url,
+        site: this._config.site,
+        env: this._config.env,
+        service: this._config.service,
+        isEvpProxy: !!this._isUsingEvpProxy,
+        ...testConfiguration
+      }
+      getSkippableSuitesRequest(configuration, callback)
+    })
+  }
+
+  /**
+   * We can't request ITR configuration until we know whether we can use the
+   * CI Visibility Protocol, hence the this._canUseCiVisProtocol promise.
+   */
+  getItrConfiguration (testConfiguration, callback) {
+    if (!this.shouldRequestItrConfiguration()) {
+      return callback(null, {})
+    }
+    this._canUseCiVisProtocolPromise.then((canUseCiVisProtocol) => {
+      if (!canUseCiVisProtocol) {
+        return callback(null, {})
+      }
+      const configuration = {
+        url: this._url,
+        env: this._config.env,
+        service: this._config.service,
+        isEvpProxy: !!this._isUsingEvpProxy,
+        ...testConfiguration
+      }
+      getItrConfigurationRequest(configuration, (err, itrConfig) => {
+        this._itrConfig = itrConfig
+        callback(err, itrConfig)
+      })
+    })
+  }
+
+  sendGitMetadata ({ url, isEvpProxy }) {
+    sendGitMetadataRequest(url, isEvpProxy, (err) => {
+      if (err) {
+        log.error(`Error uploading git metadata: ${err.message}`)
+      } else {
+        log.debug('Successfully uploaded git metadata')
+      }
+      this._resolveGit(err)
+    })
+  }
+
+  export (trace) {
+    // Until it's initialized, we just store the traces as is
+    if (!this._isInitialized) {
+      this._traceBuffer.push(trace)
+      return
+    }
+    if (!this.canReportSessionTraces() && getIsTestSessionTrace(trace)) {
+      return
+    }
+    this._export(trace)
+  }
+
+  exportCoverage (coveragePayload) {
+    // Until it's initialized, we just store the coverages as is
+    if (!this._isInitialized) {
+      this._coverageBuffer.push(coveragePayload)
+      return
+    }
+    if (!this.canReportCodeCoverage()) {
+      return
+    }
+
+    const { span, coverageFiles } = coveragePayload
+    const formattedCoverage = {
+      traceId: span.context()._traceId,
+      spanId: span.context()._spanId,
+      files: coverageFiles
+    }
+
+    this._export(formattedCoverage, this._coverageWriter, '_coverageTimer')
+  }
+
+  flush (done = () => {}) {
+    if (!this._isInitialized) {
+      return done()
+    }
+    this._writer.flush(() => {
+      if (this._coverageWriter) {
+        this._coverageWriter.flush(done)
+      } else {
+        done()
+      }
+    })
+  }
+
+  exportUncodedCoverages () {
+    this._coverageBuffer.forEach(oldCoveragePayload => {
+      this.exportCoverage(oldCoveragePayload)
+    })
+    this._coverageBuffer = []
+  }
+
+  setUrl (url, coverageUrl = url) {
+    try {
+      url = new URL(url)
+      coverageUrl = new URL(coverageUrl)
+      this._url = url
+      this._coverageUrl = coverageUrl
+      this._writer.setUrl(url)
+      this._coverageWriter.setUrl(coverageUrl)
+    } catch (e) {
+      log.error(e)
+    }
+  }
+}
+
+module.exports = CiVisibilityExporter