dd-trace 2.12.0 → 2.13.0

package/packages/dd-trace/src/plugins/util/ci.js

@@ -17,7 +17,8 @@ const {
   CI_WORKSPACE_PATH,
   CI_JOB_URL,
   CI_JOB_NAME,
-  CI_STAGE_NAME
+  CI_STAGE_NAME,
+  CI_ENV_VARS
 } = require('./tags')
 
 // Receives a string with the form 'John Doe <john.doe@gmail.com>'
@@ -104,7 +105,8 @@ module.exports = {
         GIT_BRANCH: JENKINS_GIT_BRANCH,
         GIT_COMMIT: JENKINS_GIT_COMMIT,
         GIT_URL: JENKINS_GIT_REPOSITORY_URL,
-        GIT_URL_1: JENKINS_GIT_REPOSITORY_URL_1
+        GIT_URL_1: JENKINS_GIT_REPOSITORY_URL_1,
+        DD_CUSTOM_TRACE_ID
       } = env
 
       tags = {
@@ -114,7 +116,8 @@ module.exports = {
         [CI_PROVIDER_NAME]: 'jenkins',
         [GIT_COMMIT_SHA]: JENKINS_GIT_COMMIT,
         [GIT_REPOSITORY_URL]: JENKINS_GIT_REPOSITORY_URL || JENKINS_GIT_REPOSITORY_URL_1,
-        [CI_WORKSPACE_PATH]: WORKSPACE
+        [CI_WORKSPACE_PATH]: WORKSPACE,
+        [CI_ENV_VARS]: JSON.stringify({ DD_CUSTOM_TRACE_ID })
       }
 
       const isTag = JENKINS_GIT_BRANCH && JENKINS_GIT_BRANCH.includes('tags')
@@ -152,7 +155,9 @@ module.exports = {
         CI_JOB_NAME: GITLAB_CI_JOB_NAME,
         CI_COMMIT_MESSAGE,
         CI_COMMIT_TIMESTAMP,
-        CI_COMMIT_AUTHOR
+        CI_COMMIT_AUTHOR,
+        CI_PROJECT_URL: GITLAB_PROJECT_URL,
+        CI_JOB_ID: GITLAB_CI_JOB_ID
       } = env
 
       const { name, email } = parseEmailAndName(CI_COMMIT_AUTHOR)
@@ -174,7 +179,12 @@ module.exports = {
         [GIT_COMMIT_MESSAGE]: CI_COMMIT_MESSAGE,
         [GIT_COMMIT_AUTHOR_NAME]: name,
         [GIT_COMMIT_AUTHOR_EMAIL]: email,
-        [GIT_COMMIT_AUTHOR_DATE]: CI_COMMIT_TIMESTAMP
+        [GIT_COMMIT_AUTHOR_DATE]: CI_COMMIT_TIMESTAMP,
+        [CI_ENV_VARS]: JSON.stringify({
+          CI_PROJECT_URL: GITLAB_PROJECT_URL,
+          CI_PIPELINE_ID: GITLAB_PIPELINE_ID,
+          CI_JOB_ID: GITLAB_CI_JOB_ID
+        })
       }
     }
 
@@ -188,7 +198,8 @@ module.exports = {
         CIRCLE_TAG,
         CIRCLE_SHA1,
         CIRCLE_REPOSITORY_URL,
-        CIRCLE_JOB
+        CIRCLE_JOB,
+        CIRCLE_BUILD_NUM
       } = env
 
       const pipelineUrl = `https://app.circleci.com/pipelines/workflows/${CIRCLE_WORKFLOW_ID}`
@@ -203,7 +214,11 @@ module.exports = {
         [GIT_REPOSITORY_URL]: CIRCLE_REPOSITORY_URL,
         [CI_JOB_URL]: CIRCLE_BUILD_URL,
         [CI_WORKSPACE_PATH]: CIRCLE_WORKING_DIRECTORY,
-        [CIRCLE_TAG ? GIT_TAG : GIT_BRANCH]: CIRCLE_TAG || CIRCLE_BRANCH
+        [CIRCLE_TAG ? GIT_TAG : GIT_BRANCH]: CIRCLE_TAG || CIRCLE_BRANCH,
+        [CI_ENV_VARS]: JSON.stringify({
+          CIRCLE_WORKFLOW_ID,
+          CIRCLE_BUILD_NUM
+        })
       }
     }
 
@@ -243,7 +258,13 @@ module.exports = {
         [GIT_REPOSITORY_URL]: repositoryURL,
         [CI_JOB_URL]: jobUrl,
         [CI_WORKSPACE_PATH]: GITHUB_WORKSPACE,
-        [refKey]: ref
+        [refKey]: ref,
+        [CI_ENV_VARS]: JSON.stringify({
+          GITHUB_SERVER_URL,
+          GITHUB_REPOSITORY,
+          GITHUB_RUN_ID,
+          GITHUB_RUN_ATTEMPT
+        })
       }
     }
 
@@ -443,7 +464,11 @@ module.exports = {
         [refKey]: ref,
         [GIT_COMMIT_AUTHOR_NAME]: BUILDKITE_BUILD_AUTHOR,
         [GIT_COMMIT_AUTHOR_EMAIL]: BUILDKITE_BUILD_AUTHOR_EMAIL,
-        [GIT_COMMIT_MESSAGE]: BUILDKITE_MESSAGE
+        [GIT_COMMIT_MESSAGE]: BUILDKITE_MESSAGE,
+        [CI_ENV_VARS]: JSON.stringify({
+          BUILDKITE_BUILD_ID,
+          BUILDKITE_JOB_ID
+        })
       }
     }
 

package/packages/dd-trace/src/plugins/util/git.js

@@ -1,5 +1,7 @@
-const { sanitizedExec } = require('./exec')
+const { execSync } = require('child_process')
+const os = require('os')
 
+const { sanitizedExec } = require('./exec')
 const {
   GIT_COMMIT_SHA,
   GIT_BRANCH,
@@ -15,6 +17,48 @@ const {
   CI_WORKSPACE_PATH
 } = require('./tags')
 
+function getRepositoryUrl () {
+  return sanitizedExec('git config --get remote.origin.url', { stdio: 'pipe' })
+}
+
+function getLatestCommits () {
+  return execSync('git log --format=%H -n 1000 --since="1 month ago"', { stdio: 'pipe' })
+    .toString()
+    .split('\n')
+    .filter(commit => !!commit)
+}
+
+function getCommitsToUpload (commitsToExclude) {
+  let gitCommandToGetCommitsToUpload =
+    'git rev-list --objects --no-object-names --filter=blob:none --since="1 month ago" HEAD'
+
+  commitsToExclude.forEach(commit => {
+    gitCommandToGetCommitsToUpload = `${gitCommandToGetCommitsToUpload} ^${commit}`
+  })
+
+  return execSync(gitCommandToGetCommitsToUpload, { stdio: 'pipe' })
+    .toString()
+    .split('\n')
+    .filter(commit => !!commit)
+}
+
+// Generates pack files to upload and
+// returns the ordered list of packfiles' paths
+function generatePackFilesForCommits (commitsToUpload) {
+  const tmpFolder = os.tmpdir()
+
+  const prefix = Math.floor(Math.random() * 10000)
+  const path = `${tmpFolder}/${prefix}`
+
+  const orderedCommits =
+    execSync(
+      `git pack-objects --compression=9 --max-pack-size=3m ${path}`,
+      { input: commitsToUpload.join('\n') }
+    ).toString().split('\n').filter(commit => !!commit)
+
+  return orderedCommits.map(commit => `${path}-${commit}.pack`)
+}
+
 // If there is ciMetadata, it takes precedence.
 function getGitMetadata (ciMetadata) {
   const {
@@ -57,4 +101,10 @@ function getGitMetadata (ciMetadata) {
   }
 }
 
-module.exports = { getGitMetadata }
+module.exports = {
+  getGitMetadata,
+  getLatestCommits,
+  getRepositoryUrl,
+  generatePackFilesForCommits,
+  getCommitsToUpload
+}

package/packages/dd-trace/src/plugins/util/tags.js

@@ -20,6 +20,8 @@ const CI_JOB_URL = 'ci.job.url'
 const CI_JOB_NAME = 'ci.job.name'
 const CI_STAGE_NAME = 'ci.stage.name'
 
+const CI_ENV_VARS = '_dd.ci.env_vars'
+
 module.exports = {
   GIT_COMMIT_SHA,
   GIT_BRANCH,
@@ -40,5 +42,6 @@ module.exports = {
   CI_WORKSPACE_PATH,
   CI_JOB_URL,
   CI_JOB_NAME,
-  CI_STAGE_NAME
+  CI_STAGE_NAME,
+  CI_ENV_VARS
 }

package/packages/dd-trace/src/plugins/util/web.js

@@ -2,7 +2,7 @@
 
 const uniq = require('lodash.uniq')
 const analyticsSampler = require('../../analytics_sampler')
-const FORMAT_HTTP_HEADERS = require('opentracing').FORMAT_HTTP_HEADERS
+const FORMAT_HTTP_HEADERS = 'http_headers'
 const log = require('../../log')
 const tags = require('../../../../../ext/tags')
 const types = require('../../../../../ext/types')
@@ -270,7 +270,10 @@ const web = {
   addError (req, error) {
     if (error instanceof Error) {
       const context = contexts.get(req)
-      context.error = error
+
+      if (context) {
+        context.error = error
+      }
     }
   },
 

package/packages/dd-trace/src/profiling/exporters/agent.js

@@ -2,10 +2,10 @@
 
 const retry = require('retry')
 const { request } = require('http')
-const FormData = require('./form-data')
 
 // TODO: avoid using dd-trace internals. Make this a separate module?
 const docker = require('../../exporters/common/docker')
+const FormData = require('../../exporters/common/form-data')
 const version = require('../../../../../package.json').version
 
 const containerId = docker.id()

package/packages/dd-trace/src/profiling/profilers/cpu.js

@@ -20,7 +20,7 @@ function getStartedSpans (activeSpan) {
 }
 
 function getSpanContextTags (span) {
-  return span._context()._tags
+  return span.context()._tags
 }
 
 function isWebServerSpan (tags) {
@@ -55,13 +55,13 @@ class NativeCpuProfiler {
     const active = getActiveSpan()
     if (!active) return
 
-    const activeCtx = active._context()
+    const activeCtx = active.context()
     if (!activeCtx) return
 
     const spans = getStartedSpans(active)
     if (!spans || !spans.length) return
 
-    const firstCtx = spans[0]._context()
+    const firstCtx = spans[0].context()
     if (!firstCtx) return
 
     const labels = {

package/packages/dd-trace/src/proxy.js

@@ -1,26 +1,22 @@
 'use strict'
 
-const BaseTracer = require('opentracing').Tracer
 const NoopTracer = require('./noop/tracer')
 const DatadogTracer = require('./tracer')
 const Config = require('./config')
-const Instrumenter = require('./instrumenter')
-const PluginManager = require('./plugin_manager')
 const metrics = require('./metrics')
 const log = require('./log')
 const { isFalse } = require('./util')
-const { setStartupLogInstrumenter } = require('./startup-log')
+const { setStartupLogPluginManager } = require('./startup-log')
 const telemetry = require('./telemetry')
+const PluginManager = require('./plugin_manager')
+const { sendGitMetadata } = require('./ci-visibility/exporters/git/git_metadata')
 
 const noop = new NoopTracer()
 
-class Tracer extends BaseTracer {
+class Tracer {
   constructor () {
-    super()
-
     this._initialized = false
     this._tracer = noop
-    this._instrumenter = new Instrumenter(this)
     this._pluginManager = new PluginManager(this)
     this._deprecate = method => log.deprecate(`tracer.${method}`, [
       `tracer.${method}() is deprecated.`,
@@ -61,10 +57,19 @@ class Tracer extends BaseTracer {
         }
 
         this._tracer = new DatadogTracer(config)
-        this._instrumenter.enable(config)
         this._pluginManager.configure(config)
-        setStartupLogInstrumenter(this._instrumenter)
-        telemetry.start(config, this._instrumenter, this._pluginManager)
+        setStartupLogPluginManager(this._pluginManager)
+        telemetry.start(config, this._pluginManager)
+      }
+
+      if (config.isGitUploadEnabled) {
+        sendGitMetadata(config.site, (err) => {
+          if (err) {
+            log.error(`Error uploading git metadata: ${err}`)
+          } else {
+            log.debug('Successfully uploaded git metadata')
+          }
+        })
       }
     } catch (e) {
       log.error(e)
@@ -74,7 +79,6 @@ class Tracer extends BaseTracer {
   }
 
   use () {
-    this._instrumenter.use(...arguments)
     this._pluginManager.configurePlugin(...arguments)
     return this
   }

package/packages/dd-trace/src/startup-log.js

@@ -5,38 +5,27 @@ const mainLogger = require('./log')
 const os = require('os')
 const { inspect } = require('util')
 const tracerVersion = require('../../../package.json').version
-const requirePackageJson = require('./require-package-json')
 
 const logger = Object.create(mainLogger)
 logger._enabled = true
 
 let config
-let instrumenter
+let pluginManager
 let samplingRules = []
-
 let alreadyRan = false
 
 function getIntegrationsAndAnalytics () {
   const integrations = new Set()
   const extras = {}
-  for (const plugin of instrumenter._instrumented.keys()) {
-    if (plugin.versions) {
-      try {
-        const version = requirePackageJson(plugin.name, module).version
-        integrations.add(`${plugin.name}@${version}`)
-      } catch (e) {
-        integrations.add(plugin.name)
-      }
-    } else {
-      integrations.add(plugin.name)
-    }
+  for (const pluginName in pluginManager._pluginsByName) {
+    integrations.add(pluginName)
   }
   extras.integrations_loaded = Array.from(integrations)
   return extras
 }
 
 function startupLog ({ agentError } = {}) {
-  if (!config || !instrumenter) {
+  if (!config || !pluginManager) {
     return
   }
 
@@ -106,7 +95,7 @@ function startupLog ({ agentError } = {}) {
   }
 
   config = undefined
-  instrumenter = undefined
+  pluginManager = undefined
   samplingRules = undefined
 }
 
@@ -114,8 +103,8 @@ function setStartupLogConfig (aConfig) {
   config = aConfig
 }
 
-function setStartupLogInstrumenter (theInstrumenter) {
-  instrumenter = theInstrumenter
+function setStartupLogPluginManager (thePluginManager) {
+  pluginManager = thePluginManager
 }
 
 function setSamplingRules (theRules) {
@@ -125,6 +114,6 @@ function setSamplingRules (theRules) {
 module.exports = {
   startupLog,
   setStartupLogConfig,
-  setStartupLogInstrumenter,
+  setStartupLogPluginManager,
   setSamplingRules
 }

package/packages/dd-trace/src/telemetry.js

@@ -9,7 +9,6 @@ const os = require('os')
 const request = require('./exporters/common/request')
 
 let config
-let instrumenter
 let pluginManager
 
 let seqId = 0
@@ -20,17 +19,6 @@ const sentIntegrations = new Set()
 
 function getIntegrations () {
   const newIntegrations = []
-  for (const plugin of instrumenter._instrumented.keys()) {
-    if (sentIntegrations.has(plugin.name)) {
-      continue
-    }
-    newIntegrations.push({
-      name: plugin.name,
-      enabled: true,
-      auto_enabled: true
-    })
-    sentIntegrations.add(plugin.name)
-  }
   for (const pluginName in pluginManager._pluginsByName) {
     if (sentIntegrations.has(pluginName)) {
       continue
@@ -146,12 +134,11 @@ function sendData (reqType, payload = {}) {
   })
 }
 
-function start (aConfig, theInstrumenter, thePluginManager) {
+function start (aConfig, thePluginManager) {
   if (!aConfig.telemetryEnabled) {
     return
   }
   config = aConfig
-  instrumenter = theInstrumenter
   pluginManager = thePluginManager
   application = createAppObject()
   host = createHostObject()

package/scripts/install_plugin_modules.js

@@ -7,8 +7,6 @@ const semver = require('semver')
 const proxyquire = require('proxyquire')
 const exec = require('./helpers/exec')
 const childProcess = require('child_process')
-const plugins = require('../packages/dd-trace/src/plugins')
-const Plugin = require('../packages/dd-trace/src/plugins/plugin')
 const externals = require('../packages/dd-trace/test/plugins/externals')
 
 const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require-package-json')
@@ -16,6 +14,9 @@ const requirePackageJsonPath = require.resolve('../packages/dd-trace/src/require
 const workspaces = new Set()
 const versionLists = {}
 const deps = {}
+const names = []
+const filter = process.env.hasOwnProperty('PLUGINS') && process.env.PLUGINS.split('|')
+
 Object.keys(externals).forEach(external => externals[external].forEach(thing => {
   if (thing.dep) {
     if (!deps[external]) {
@@ -29,7 +30,10 @@ fs.readdirSync(path.join(__dirname, '../packages/datadog-instrumentations/src'))
   .filter(file => file.endsWith('js'))
   .forEach(file => {
     file = file.replace('.js', '')
-    plugins[file] = { name: file, prototype: Object.create(Plugin.prototype) }
+
+    if (!filter || filter.includes(file)) {
+      names.push(file)
+    }
   })
 
 run()
@@ -42,32 +46,19 @@ async function run () {
 }
 
 async function assertVersions () {
-  let filter = []
-  let names = Object.keys(plugins)
-
-  if (process.env.hasOwnProperty('PLUGINS')) {
-    filter = process.env.PLUGINS.split('|')
-    names = names.filter(name => ~filter.indexOf(name))
-  }
-
   const internals = names
     .map(key => {
-      const plugin = plugins[key]
-      if (plugin.prototype instanceof Plugin) {
-        const instrumentations = []
-        const name = plugin.name
-
-        try {
-          loadInstFile(`${name}/server.js`, instrumentations)
-          loadInstFile(`${name}/client.js`, instrumentations)
-        } catch (e) {
-          loadInstFile(`${name}.js`, instrumentations)
-        }
-
-        return instrumentations
-      } else {
-        return plugin
+      const instrumentations = []
+      const name = key
+
+      try {
+        loadInstFile(`${name}/server.js`, instrumentations)
+        loadInstFile(`${name}/client.js`, instrumentations)
+      } catch (e) {
+        loadInstFile(`${name}.js`, instrumentations)
       }
+
+      return instrumentations
     })
     .reduce((prev, next) => prev.concat(next), [])