dd-trace 3.44.0 → 3.46.0

package/packages/dd-trace/src/profiling/profilers/wall.js

@@ -7,7 +7,7 @@ const { HTTP_METHOD, HTTP_ROUTE, RESOURCE_NAME, SPAN_TYPE } = require('../../../
 const { WEB } = require('../../../../../ext/types')
 const runtimeMetrics = require('../../runtime_metrics')
 const telemetryMetrics = require('../../telemetry/metrics')
-const { END_TIMESTAMP_LABEL, getThreadLabels } = require('./shared')
+const { END_TIMESTAMP_LABEL, getNonJSThreadsLabels, getThreadLabels } = require('./shared')
 
 const beforeCh = dc.channel('dd-trace:storage:before')
 const enterCh = dc.channel('dd-trace:storage:enter')
@@ -78,13 +78,15 @@ class NativeWallProfiler {
     this._codeHotspotsEnabled = !!options.codeHotspotsEnabled
     this._endpointCollectionEnabled = !!options.endpointCollectionEnabled
     this._timelineEnabled = !!options.timelineEnabled
+    this._cpuProfilingEnabled = !!options.cpuProfilingEnabled
     // We need to capture span data into the sample context for either code hotspots
     // or endpoint collection.
     this._captureSpanData = this._codeHotspotsEnabled || this._endpointCollectionEnabled
     // We need to run the pprof wall profiler with sample contexts if we're either
     // capturing span data or timeline is enabled (so we need sample timestamps, and for now
-    // timestamps require the sample contexts feature in the pprof wall profiler.)
-    this._withContexts = this._captureSpanData || this._timelineEnabled
+    // timestamps require the sample contexts feature in the pprof wall profiler), or
+    // cpu profiling is enabled.
+    this._withContexts = this._captureSpanData || this._timelineEnabled || this._cpuProfilingEnabled
     this._v8ProfilerBugWorkaroundEnabled = !!options.v8ProfilerBugWorkaroundEnabled
     this._mapper = undefined
     this._pprof = undefined
@@ -131,7 +133,8 @@ class NativeWallProfiler {
       sourceMapper: this._mapper,
       withContexts: this._withContexts,
       lineNumbers: false,
-      workaroundV8Bug: this._v8ProfilerBugWorkaroundEnabled
+      workaroundV8Bug: this._v8ProfilerBugWorkaroundEnabled,
+      collectCpuTime: this._cpuProfilingEnabled
     })
 
     if (this._withContexts) {
@@ -220,22 +223,42 @@ class NativeWallProfiler {
 
   _stop (restart) {
     if (!this._started) return
+
     if (this._captureSpanData) {
       // update last sample context if needed
       this._enter()
       this._lastSampleCount = 0
     }
     const profile = this._pprof.time.stop(restart, this._generateLabels)
+
     if (restart) {
       const v8BugDetected = this._pprof.time.v8ProfilerStuckEventLoopDetected()
       if (v8BugDetected !== 0) {
         this._reportV8bug(v8BugDetected === 1)
       }
+    } else {
+      if (this._captureSpanData) {
+        beforeCh.unsubscribe(this._enter)
+        enterCh.unsubscribe(this._enter)
+        spanFinishCh.unsubscribe(this._spanFinished)
+        this._profilerState = undefined
+        this._lastSpan = undefined
+        this._lastStartedSpans = undefined
+        this._lastWebTags = undefined
+      }
+      this._started = false
     }
+
     return profile
   }
 
-  _generateLabels (context) {
+  _generateLabels ({ node, context }) {
+    // check for special node that represents CPU time all non-JS threads.
+    // In that case only return a special thread name label since we cannot associate any timestamp/span/endpoint to it.
+    if (node.name === this._pprof.time.constants.NON_JS_THREADS_FUNCTION_NAME) {
+      return getNonJSThreadsLabels()
+    }
+
     if (context == null) {
       // generateLabels is also called for samples without context.
       // In that case just return thread labels.
@@ -267,8 +290,8 @@ class NativeWallProfiler {
     return labels
   }
 
-  profile () {
-    return this._stop(true)
+  profile (restart) {
+    return this._stop(restart)
   }
 
   encode (profile) {
@@ -276,21 +299,11 @@ class NativeWallProfiler {
   }
 
   stop () {
-    if (!this._started) return
-
-    const profile = this._stop(false)
-    if (this._captureSpanData) {
-      beforeCh.unsubscribe(this._enter)
-      enterCh.unsubscribe(this._enter)
-      spanFinishCh.unsubscribe(this._spanFinished)
-      this._profilerState = undefined
-      this._lastSpan = undefined
-      this._lastStartedSpans = undefined
-      this._lastWebTags = undefined
-    }
+    this._stop(false)
+  }
 
-    this._started = false
-    return profile
+  isStarted () {
+    return this._started
   }
 }
 

package/packages/dd-trace/src/telemetry/index.js

@@ -140,8 +140,7 @@ function appStarted (config) {
   return app
 }
 
-function onBeforeExit () {
-  process.removeListener('beforeExit', onBeforeExit)
+function appClosing () {
   const { reqType, payload } = createPayload('app-closing')
   sendData(config, application, host, reqType, payload)
   // we flush before shutting down. Only in CI Visibility
@@ -150,6 +149,11 @@ function onBeforeExit () {
   }
 }
 
+function onBeforeExit () {
+  process.removeListener('beforeExit', onBeforeExit)
+  appClosing()
+}
+
 function createAppObject (config) {
   return {
     service_name: config.service,
@@ -339,5 +343,6 @@ module.exports = {
   start,
   stop,
   updateIntegrations,
-  updateConfig
+  updateConfig,
+  appClosing
 }

package/packages/dd-trace/src/telemetry/send-data.js

@@ -1,6 +1,8 @@
 
 const request = require('../exporters/common/request')
 const log = require('../log')
+const { isTrue } = require('../util')
+
 let agentTelemetry = true
 
 function getHeaders (config, application, reqType) {
@@ -15,9 +17,22 @@ function getHeaders (config, application, reqType) {
   if (debug) {
     headers['dd-telemetry-debug-enabled'] = 'true'
   }
+  if (config.apiKey) {
+    headers['dd-api-key'] = config.apiKey
+  }
   return headers
 }
 
+function getAgentlessTelemetryEndpoint (site) {
+  if (site === 'datad0g.com') { // staging
+    return 'https://all-http-intake.logs.datad0g.com'
+  }
+  if (site === 'datadoghq.eu') {
+    return 'https://instrumentation-telemetry-intake.eu1.datadoghq.com'
+  }
+  return `https://instrumentation-telemetry-intake.${site}`
+}
+
 let seqId = 0
 
 function getPayload (payload) {
@@ -35,17 +50,32 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
   const {
     hostname,
     port,
-    url
+    isCiVisibility
   } = config
 
+  let url = config.url
+
+  const isCiVisibilityAgentlessMode = isCiVisibility && isTrue(process.env.DD_CIVISIBILITY_AGENTLESS_ENABLED)
+
+  if (isCiVisibilityAgentlessMode) {
+    try {
+      url = url || new URL(getAgentlessTelemetryEndpoint(config.site))
+    } catch (err) {
+      log.error(err)
+      // No point to do the request if the URL is invalid
+      return cb(err, { payload, reqType })
+    }
+  }
+
   const options = {
     url,
     hostname,
     port,
     method: 'POST',
-    path: '/telemetry/proxy/api/v2/apmtelemetry',
+    path: isCiVisibilityAgentlessMode ? '/api/v2/apmtelemetry' : '/telemetry/proxy/api/v2/apmtelemetry',
     headers: getHeaders(config, application, reqType)
   }
+
   const data = JSON.stringify({
     api_version: 'v2',
     naming_schema_version: config.spanAttributeSchema ? config.spanAttributeSchema : '',
@@ -65,24 +95,13 @@ function sendData (config, application, host, reqType, payload = {}, cb = () =>
         agentTelemetry = false
       }
       // figure out which data center to send to
-      let backendUrl
-      const dataCenters = [
-        'datadoghq.com',
-        'us3.datadoghq.com',
-        'us5.datadoghq.com',
-        'ap1.datadoghq.com',
-        'eu1.datadoghq.com'
-      ]
-      if (config.site === 'datad0g.com') { // staging
-        backendUrl = 'https://all-http-intake.logs.datad0g.com/api/v2/apmtelemetry'
-      } else if (dataCenters.includes(config.site)) {
-        backendUrl = 'https://instrumentation-telemetry-intake.' + config.site + '/api/v2/apmtelemetry'
-      }
+      const backendUrl = getAgentlessTelemetryEndpoint(config.site)
       const backendHeader = { ...options.headers, 'DD-API-KEY': process.env.DD_API_KEY }
       const backendOptions = {
         ...options,
         url: backendUrl,
-        headers: backendHeader
+        headers: backendHeader,
+        path: '/api/v2/apmtelemetry'
       }
       if (backendUrl) {
         request(data, backendOptions, (error) => { log.error(error) })

package/scripts/st.js

@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+/* eslint-disable no-console, no-fallthrough */
+'use strict'
+
+const path = require('path')
+const { writeFileSync } = require('fs')
+const { execSync } = require('child_process')
+
+const ddtracePath = path.join(__dirname, '..')
+const defaultTestPath = process.env.DD_ST_PATH || path.join(ddtracePath, '..', 'system-tests')
+
+const { buildAll, npm, testDir, testArgs } = parseArgs()
+
+const binariesPath = path.join(testDir, 'binaries')
+
+if (npm) {
+  console.log('Using NPM package:', npm)
+
+  writeFileSync(path.join(binariesPath, 'nodejs-load-from-npm'), npm)
+} else {
+  console.log('Using local repo')
+
+  const packName = execSync(`npm pack ${ddtracePath}`, {
+    cwd: binariesPath,
+    stdio: [null, null, 'inherit'],
+    encoding: 'utf8'
+  }).slice(0, -1) // remove trailing newline
+
+  writeFileSync(path.join(binariesPath, 'nodejs-load-from-npm'), `/binaries/${packName}`)
+}
+
+try {
+  execSync(`./build.sh ${buildAll ? '' : '-i weblog'} && ./run.sh ${testArgs}`, {
+    cwd: testDir,
+    stdio: [null, 'inherit', 'inherit']
+  })
+} catch (err) {
+  process.exit(err.status || 1)
+}
+
+function parseArgs () {
+  const args = {
+    buildAll: false,
+    npm: null,
+    testDir: defaultTestPath,
+    testArgs: ''
+  }
+
+  for (let i = 2; i < process.argv.length; i++) {
+    switch (process.argv[i]) {
+      case '-b':
+      case '--build-all':
+        args.buildAll = true
+        break
+
+      case '-h':
+      case '--help':
+        helpAndExit()
+        break
+
+      case '-n':
+      case '--npm': {
+        const arg = process.argv[i + 1]
+        if (!arg || arg[0] === '-') {
+          args.npm = 'dd-trace'
+        } else {
+          args.npm = arg
+          i++
+        }
+        break
+      }
+
+      case '-t':
+      case '--test-dir': {
+        const arg = process.argv[++i]
+        if (!arg || arg[0] === '-') helpAndExit()
+        args.testDir = arg
+        break
+      }
+
+      case '--':
+        args.testArgs = process.argv.slice(i + 1).join(' ')
+        return args
+
+      default:
+        console.log('Unknown option:', process.argv[i], '\n')
+        helpAndExit()
+    }
+  }
+
+  return args
+}
+
+function helpAndExit () {
+  console.log('Usage: node st.js [options...] [-- test-args]')
+  console.log('Options:')
+  console.log('  -b, --build-all       Rebuild all images (default: only build weblog)')
+  console.log('  -h, --help            Print this message')
+  console.log('  -n, --npm [package]   Build a remote package instead of the local repo (default: "dd-trace")')
+  console.log('                        Can be a package name (e.g. "dd-trace@4.2.0") or a git URL (e.g.')
+  console.log('                        "git+https://github.com/DataDog/dd-trace-js.git#mybranch")')
+  console.log('  -t, --test-dir <path> Specify the system-tests directory (default: "dd-trace/../system-tests/")')
+  console.log('  -- <test-args>        Passed to system-tests run.sh (e.g. "-- SCENARIO_NAME tests/path_to_test.py")')
+  process.exit()
+}