dd-trace 4.47.1 → 4.49.0

package/packages/dd-trace/src/azure_metadata.js

@@ -0,0 +1,120 @@
+'use strict'
+
+// eslint-disable-next-line max-len
+// Modeled after https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/ddcommon/src/azure_app_services.rs
+
+const os = require('os')
+const { getIsAzureFunction } = require('./serverless')
+
+function extractSubscriptionID (ownerName) {
+  if (ownerName !== undefined) {
+    const subId = ownerName.split('+')[0].trim()
+    if (subId.length > 0) {
+      return subId
+    }
+  }
+  return undefined
+}
+
+function extractResourceGroup (ownerName) {
+  return /.+\+(.+)-.+webspace(-Linux)?/.exec(ownerName)?.[1]
+}
+
+function buildResourceID (subscriptionID, siteName, resourceGroup) {
+  if (subscriptionID === undefined || siteName === undefined || resourceGroup === undefined) {
+    return undefined
+  }
+  return `/subscriptions/${subscriptionID}/resourcegroups/${resourceGroup}/providers/microsoft.web/sites/${siteName}`
+    .toLowerCase()
+}
+
+function trimObject (obj) {
+  Object.entries(obj)
+    .filter(([_, value]) => value === undefined)
+    .forEach(([key, _]) => { delete obj[key] })
+  return obj
+}
+
+function buildMetadata () {
+  const {
+    COMPUTERNAME,
+    DD_AAS_DOTNET_EXTENSION_VERSION,
+    FUNCTIONS_EXTENSION_VERSION,
+    FUNCTIONS_WORKER_RUNTIME,
+    FUNCTIONS_WORKER_RUNTIME_VERSION,
+    WEBSITE_INSTANCE_ID,
+    WEBSITE_OWNER_NAME,
+    WEBSITE_OS,
+    WEBSITE_RESOURCE_GROUP,
+    WEBSITE_SITE_NAME
+  } = process.env
+
+  const subscriptionID = extractSubscriptionID(WEBSITE_OWNER_NAME)
+
+  const siteName = WEBSITE_SITE_NAME
+
+  const [siteKind, siteType] = getIsAzureFunction()
+    ? ['functionapp', 'function']
+    : ['app', 'app']
+
+  const resourceGroup = WEBSITE_RESOURCE_GROUP ?? extractResourceGroup(WEBSITE_OWNER_NAME)
+
+  return trimObject({
+    extensionVersion: DD_AAS_DOTNET_EXTENSION_VERSION,
+    functionRuntimeVersion: FUNCTIONS_EXTENSION_VERSION,
+    instanceID: WEBSITE_INSTANCE_ID,
+    instanceName: COMPUTERNAME,
+    operatingSystem: WEBSITE_OS ?? os.platform(),
+    resourceGroup,
+    resourceID: buildResourceID(subscriptionID, siteName, resourceGroup),
+    runtime: FUNCTIONS_WORKER_RUNTIME,
+    runtimeVersion: FUNCTIONS_WORKER_RUNTIME_VERSION,
+    siteKind,
+    siteName,
+    siteType,
+    subscriptionID
+  })
+}
+
+function getAzureAppMetadata () {
+  // DD_AZURE_APP_SERVICES is an environment variable introduced by the .NET APM team and is set automatically for
+  // anyone using the Datadog APM Extensions (.NET, Java, or Node) for Windows Azure App Services
+  // eslint-disable-next-line max-len
+  // See: https://github.com/DataDog/datadog-aas-extension/blob/01f94b5c28b7fa7a9ab264ca28bd4e03be603900/node/src/applicationHost.xdt#L20-L21
+  return process.env.DD_AZURE_APP_SERVICES !== undefined ? buildMetadata() : undefined
+}
+
+function getAzureFunctionMetadata () {
+  return getIsAzureFunction() ? buildMetadata() : undefined
+}
+
+// eslint-disable-next-line max-len
+// Modeled after https://github.com/DataDog/libdatadog/blob/92272e90a7919f07178f3246ef8f82295513cfed/profiling/src/exporter/mod.rs#L187
+// eslint-disable-next-line max-len
+// and https://github.com/DataDog/libdatadog/blob/f3994857a59bb5679a65967138c5a3aec418a65f/trace-utils/src/trace_utils.rs#L533
+function getAzureTagsFromMetadata (metadata) {
+  if (metadata === undefined) {
+    return {}
+  }
+  return trimObject({
+    'aas.environment.extension_version': metadata.extensionVersion,
+    'aas.environment.function_runtime': metadata.functionRuntimeVersion,
+    'aas.environment.instance_id': metadata.instanceID,
+    'aas.environment.instance_name': metadata.instanceName,
+    'aas.environment.os': metadata.operatingSystem,
+    'aas.environment.runtime': metadata.runtime,
+    'aas.environment.runtime_version': metadata.runtimeVersion,
+    'aas.resource.group': metadata.resourceGroup,
+    'aas.resource.id': metadata.resourceID,
+    'aas.site.kind': metadata.siteKind,
+    'aas.site.name': metadata.siteName,
+    'aas.site.type': metadata.siteType,
+    'aas.subscription.id': metadata.subscriptionID
+  })
+}
+
+module.exports = {
+  getAzureAppMetadata,
+  getAzureFunctionMetadata,
+  getAzureTagsFromMetadata
+}

package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/index.js

@@ -0,0 +1,97 @@
+'use strict'
+
+const { join } = require('path')
+const { Worker } = require('worker_threads')
+const { randomUUID } = require('crypto')
+const log = require('../../log')
+
+const probeIdToResolveBreakpointSet = new Map()
+const probeIdToResolveBreakpointHit = new Map()
+
+class TestVisDynamicInstrumentation {
+  constructor () {
+    this.worker = null
+    this._readyPromise = new Promise(resolve => {
+      this._onReady = resolve
+    })
+    this.breakpointSetChannel = new MessageChannel()
+    this.breakpointHitChannel = new MessageChannel()
+  }
+
+  // Return 3 elements:
+  // 1. Snapshot ID
+  // 2. Promise that's resolved when the breakpoint is set
+  // 3. Promise that's resolved when the breakpoint is hit
+  addLineProbe ({ file, line }) {
+    const snapshotId = randomUUID()
+    const probeId = randomUUID()
+
+    this.breakpointSetChannel.port2.postMessage({
+      snapshotId,
+      probe: { id: probeId, file, line }
+    })
+
+    return [
+      snapshotId,
+      new Promise(resolve => {
+        probeIdToResolveBreakpointSet.set(probeId, resolve)
+      }),
+      new Promise(resolve => {
+        probeIdToResolveBreakpointHit.set(probeId, resolve)
+      })
+    ]
+  }
+
+  isReady () {
+    return this._readyPromise
+  }
+
+  start () {
+    if (this.worker) return
+
+    const { NODE_OPTIONS, ...envWithoutNodeOptions } = process.env
+
+    log.debug('Starting Test Visibility - Dynamic Instrumentation client...')
+
+    this.worker = new Worker(
+      join(__dirname, 'worker', 'index.js'),
+      {
+        execArgv: [],
+        env: envWithoutNodeOptions,
+        workerData: {
+          breakpointSetChannel: this.breakpointSetChannel.port1,
+          breakpointHitChannel: this.breakpointHitChannel.port1
+        },
+        transferList: [this.breakpointSetChannel.port1, this.breakpointHitChannel.port1]
+      }
+    )
+    this.worker.on('online', () => {
+      log.debug('Test Visibility - Dynamic Instrumentation client is ready')
+      this._onReady()
+    })
+
+    // Allow the parent to exit even if the worker is still running
+    this.worker.unref()
+
+    this.breakpointSetChannel.port2.on('message', (message) => {
+      const { probeId } = message
+      const resolve = probeIdToResolveBreakpointSet.get(probeId)
+      if (resolve) {
+        resolve()
+        probeIdToResolveBreakpointSet.delete(probeId)
+      }
+    }).unref()
+
+    this.breakpointHitChannel.port2.on('message', (message) => {
+      const { snapshot } = message
+      const { probe: { id: probeId } } = snapshot
+      const resolve = probeIdToResolveBreakpointHit.get(probeId)
+      if (resolve) {
+        resolve({ snapshot })
+        probeIdToResolveBreakpointHit.delete(probeId)
+      }
+    }).unref()
+  }
+}
+
+module.exports = new TestVisDynamicInstrumentation()

package/packages/dd-trace/src/ci-visibility/dynamic-instrumentation/worker/index.js

@@ -0,0 +1,90 @@
+'use strict'
+
+const { workerData: { breakpointSetChannel, breakpointHitChannel } } = require('worker_threads')
+// TODO: move debugger/devtools_client/session to common place
+const session = require('../../../debugger/devtools_client/session')
+// TODO: move debugger/devtools_client/snapshot to common place
+const { getLocalStateForCallFrame } = require('../../../debugger/devtools_client/snapshot')
+// TODO: move debugger/devtools_client/state to common place
+const {
+  findScriptFromPartialPath,
+  getStackFromCallFrames
+} = require('../../../debugger/devtools_client/state')
+const log = require('../../../log')
+
+let sessionStarted = false
+
+const breakpointIdToSnapshotId = new Map()
+const breakpointIdToProbe = new Map()
+
+session.on('Debugger.paused', async ({ params: { hitBreakpoints: [hitBreakpoint], callFrames } }) => {
+  const probe = breakpointIdToProbe.get(hitBreakpoint)
+  if (!probe) {
+    log.warn(`No probe found for breakpoint ${hitBreakpoint}`)
+    return session.post('Debugger.resume')
+  }
+
+  const stack = getStackFromCallFrames(callFrames)
+
+  const getLocalState = await getLocalStateForCallFrame(callFrames[0])
+
+  await session.post('Debugger.resume')
+
+  const snapshotId = breakpointIdToSnapshotId.get(hitBreakpoint)
+
+  const snapshot = {
+    id: snapshotId,
+    timestamp: Date.now(),
+    probe: {
+      id: probe.probeId,
+      version: '0',
+      location: probe.location
+    },
+    stack,
+    language: 'javascript'
+  }
+
+  const state = getLocalState()
+  if (state) {
+    snapshot.captures = {
+      lines: { [probe.location.lines[0]]: { locals: state } }
+    }
+  }
+
+  breakpointHitChannel.postMessage({ snapshot })
+})
+
+// TODO: add option to remove breakpoint
+breakpointSetChannel.on('message', async ({ snapshotId, probe: { id: probeId, file, line } }) => {
+  await addBreakpoint(snapshotId, { probeId, file, line })
+  breakpointSetChannel.postMessage({ probeId })
+})
+
+async function addBreakpoint (snapshotId, probe) {
+  if (!sessionStarted) await start()
+  const { file, line } = probe
+
+  probe.location = { file, lines: [String(line)] }
+
+  const script = findScriptFromPartialPath(file)
+  if (!script) throw new Error(`No loaded script found for ${file}`)
+
+  const [path, scriptId] = script
+
+  log.debug(`Adding breakpoint at ${path}:${line}`)
+
+  const { breakpointId } = await session.post('Debugger.setBreakpoint', {
+    location: {
+      scriptId,
+      lineNumber: line - 1
+    }
+  })
+
+  breakpointIdToProbe.set(breakpointId, probe)
+  breakpointIdToSnapshotId.set(breakpointId, snapshotId)
+}
+
+function start () {
+  sessionStarted = true
+  return session.post('Debugger.enable') // return instead of await to reduce number of promises created
+}

package/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js

@@ -12,19 +12,7 @@ const {
   TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES
 } = require('../../ci-visibility/telemetry')
 
-function getNumTests (knownTests) {
-  let totalNumTests = 0
-
-  for (const testModule of Object.values(knownTests)) {
-    for (const testSuite of Object.values(testModule)) {
-      for (const testList of Object.values(testSuite)) {
-        totalNumTests += testList.length
-      }
-    }
-  }
-
-  return totalNumTests
-}
+const { getNumFromKnownTests } = require('../../plugins/util/test')
 
 function getKnownTests ({
   url,
@@ -102,7 +90,7 @@ function getKnownTests ({
       try {
         const { data: { attributes: { tests: knownTests } } } = JSON.parse(res)
 
-        const numTests = getNumTests(knownTests)
+        const numTests = getNumFromKnownTests(knownTests)
 
         incrementCountMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, numTests)
         distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, res.length)

package/packages/dd-trace/src/ci-visibility/exporters/agent-proxy/index.js

@@ -7,6 +7,7 @@ const CiVisibilityExporter = require('../ci-visibility-exporter')
 
 const AGENT_EVP_PROXY_PATH_PREFIX = '/evp_proxy/v'
 const AGENT_EVP_PROXY_PATH_REGEX = /\/evp_proxy\/v(\d+)\/?/
+const AGENT_DEBUGGER_INPUT = '/debugger/v1/input'
 
 function getLatestEvpProxyVersion (err, agentInfo) {
   if (err) {
@@ -24,6 +25,10 @@ function getLatestEvpProxyVersion (err, agentInfo) {
   }, 0)
 }
 
+function getCanForwardDebuggerLogs (err, agentInfo) {
+  return !err && agentInfo.endpoints.some(endpoint => endpoint === AGENT_DEBUGGER_INPUT)
+}
+
 class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
   constructor (config) {
     super(config)
@@ -33,7 +38,8 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
       prioritySampler,
       lookup,
       protocolVersion,
-      headers
+      headers,
+      isTestDynamicInstrumentationEnabled
     } = config
 
     this.getAgentInfo((err, agentInfo) => {
@@ -60,6 +66,18 @@ class AgentProxyCiVisibilityExporter extends CiVisibilityExporter {
           url: this._url,
           evpProxyPrefix
         })
+        if (isTestDynamicInstrumentationEnabled) {
+          const canFowardLogs = getCanForwardDebuggerLogs(err, agentInfo)
+          if (canFowardLogs) {
+            const DynamicInstrumentationLogsWriter = require('../agentless/di-logs-writer')
+            this._logsWriter = new DynamicInstrumentationLogsWriter({
+              url: this._url,
+              tags,
+              isAgentProxy: true
+            })
+            this._canForwardLogs = true
+          }
+        }
       } else {
         this._writer = new AgentWriter({
           url: this._url,

package/packages/dd-trace/src/ci-visibility/exporters/agentless/di-logs-writer.js

@@ -0,0 +1,53 @@
+'use strict'
+const request = require('../../../exporters/common/request')
+const log = require('../../../log')
+const { safeJSONStringify } = require('../../../exporters/common/util')
+const { JSONEncoder } = require('../../encode/json-encoder')
+
+const BaseWriter = require('../../../exporters/common/writer')
+
+// Writer used by the integration between Dynamic Instrumentation and Test Visibility
+// It is used to encode and send logs to both the logs intake directly and the
+// `/debugger/v1/input` endpoint in the agent, which is a proxy to the logs intake.
+class DynamicInstrumentationLogsWriter extends BaseWriter {
+  constructor ({ url, timeout, isAgentProxy = false }) {
+    super(...arguments)
+    this._url = url
+    this._encoder = new JSONEncoder()
+    this._isAgentProxy = isAgentProxy
+    this.timeout = timeout
+  }
+
+  _sendPayload (data, _, done) {
+    const options = {
+      path: '/api/v2/logs',
+      method: 'POST',
+      headers: {
+        'dd-api-key': process.env.DATADOG_API_KEY || process.env.DD_API_KEY,
+        'Content-Type': 'application/json'
+      },
+      // TODO: what's a good value for timeout for the logs intake?
+      timeout: this.timeout || 15000,
+      url: this._url
+    }
+
+    if (this._isAgentProxy) {
+      delete options.headers['dd-api-key']
+      options.path = '/debugger/v1/input'
+    }
+
+    log.debug(() => `Request to the logs intake: ${safeJSONStringify(options)}`)
+
+    request(data, options, (err, res) => {
+      if (err) {
+        log.error(err)
+        done()
+        return
+      }
+      log.debug(`Response from the logs intake: ${res}`)
+      done()
+    })
+  }
+}
+
+module.exports = DynamicInstrumentationLogsWriter

package/packages/dd-trace/src/ci-visibility/exporters/agentless/index.js

@@ -9,10 +9,11 @@ const log = require('../../../log')
 class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
   constructor (config) {
     super(config)
-    const { tags, site, url } = config
+    const { tags, site, url, isTestDynamicInstrumentationEnabled } = config
     // we don't need to request /info because we are using agentless by configuration
     this._isInitialized = true
     this._resolveCanUseCiVisProtocol(true)
+    this._canForwardLogs = true
 
     this._url = url || new URL(`https://citestcycle-intake.${site}`)
     this._writer = new Writer({ url: this._url, tags })
@@ -20,6 +21,12 @@ class AgentlessCiVisibilityExporter extends CiVisibilityExporter {
     this._coverageUrl = url || new URL(`https://citestcov-intake.${site}`)
     this._coverageWriter = new CoverageWriter({ url: this._coverageUrl })
 
+    if (isTestDynamicInstrumentationEnabled) {
+      const DynamicInstrumentationLogsWriter = require('./di-logs-writer')
+      this._logsUrl = url || new URL(`https://http-intake.logs.${site}`)
+      this._logsWriter = new DynamicInstrumentationLogsWriter({ url: this._logsUrl, tags })
+    }
+
     this._apiUrl = url || new URL(`https://api.${site}`)
     // Agentless is always gzip compatible
     this._isGzipCompatible = true

package/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js

@@ -8,6 +8,7 @@ const { getSkippableSuites: getSkippableSuitesRequest } = require('../intelligen
 const { getKnownTests: getKnownTestsRequest } = require('../early-flake-detection/get-known-tests')
 const log = require('../../log')
 const AgentInfoExporter = require('../../exporters/common/agent-info-exporter')
+const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags')
 
 function getTestConfigurationTags (tags) {
   if (!tags) {
@@ -36,6 +37,7 @@ class CiVisibilityExporter extends AgentInfoExporter {
     super(config)
     this._timer = undefined
     this._coverageTimer = undefined
+    this._logsTimer = undefined
     this._coverageBuffer = []
     // The library can use new features like ITR and test suite level visibility
     // AKA CI Vis Protocol
@@ -255,6 +257,47 @@ class CiVisibilityExporter extends AgentInfoExporter {
     this._export(formattedCoverage, this._coverageWriter, '_coverageTimer')
   }
 
+  formatLogMessage (testConfiguration, logMessage) {
+    const {
+      [GIT_REPOSITORY_URL]: gitRepositoryUrl,
+      [GIT_COMMIT_SHA]: gitCommitSha
+    } = testConfiguration
+
+    const { service, env, version } = this._config
+
+    return {
+      ddtags: [
+        ...(logMessage.ddtags || []),
+        `${GIT_REPOSITORY_URL}:${gitRepositoryUrl}`,
+        `${GIT_COMMIT_SHA}:${gitCommitSha}`
+      ].join(','),
+      level: 'error',
+      service,
+      dd: {
+        ...(logMessage.dd || []),
+        service,
+        env,
+        version
+      },
+      ddsource: 'dd_debugger',
+      ...logMessage
+    }
+  }
+
+  // DI logs
+  exportDiLogs (testConfiguration, logMessage) {
+    // TODO: could we lose logs if it's not initialized?
+    if (!this._config.isTestDynamicInstrumentationEnabled || !this._isInitialized || !this._canForwardLogs) {
+      return
+    }
+
+    this._export(
+      this.formatLogMessage(testConfiguration, logMessage),
+      this._logsWriter,
+      '_logsTimer'
+    )
+  }
+
   flush (done = () => {}) {
     if (!this._isInitialized) {
       return done()

package/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js

@@ -0,0 +1,53 @@
+const Plugin = require('../../plugins/plugin')
+const log = require('../../log')
+
+function getWinstonLogSubmissionParameters (config) {
+  const { site, service } = config
+
+  const defaultParameters = {
+    host: `http-intake.logs.${site}`,
+    path: `/api/v2/logs?ddsource=winston&service=${service}`,
+    ssl: true,
+    headers: {
+      'DD-API-KEY': process.env.DD_API_KEY
+    }
+  }
+
+  if (!process.env.DD_AGENTLESS_LOG_SUBMISSION_URL) {
+    return defaultParameters
+  }
+
+  try {
+    const url = new URL(process.env.DD_AGENTLESS_LOG_SUBMISSION_URL)
+    return {
+      host: url.hostname,
+      port: url.port,
+      ssl: url.protocol === 'https:',
+      path: defaultParameters.path,
+      headers: defaultParameters.headers
+    }
+  } catch (e) {
+    log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL')
+    return defaultParameters
+  }
+}
+
+class LogSubmissionPlugin extends Plugin {
+  static get id () {
+    return 'log-submission'
+  }
+
+  constructor (...args) {
+    super(...args)
+
+    this.addSub('ci:log-submission:winston:configure', (httpClass) => {
+      this.HttpClass = httpClass
+    })
+
+    this.addSub('ci:log-submission:winston:add-transport', (logger) => {
+      logger.add(new this.HttpClass(getWinstonLogSubmissionParameters(this.config)))
+    })
+  }
+}
+
+module.exports = LogSubmissionPlugin