@depup/dd-trace 5.90.0-depup.0

package/packages/dd-trace/src/plugins/util/git.js

@@ -0,0 +1,613 @@
+'use strict'
+
+const os = require('os')
+const path = require('path')
+const fs = require('fs')
+
+const log = require('../../log')
+const {
+  incrementCountMetric,
+  distributionMetric,
+  TELEMETRY_GIT_COMMAND,
+  TELEMETRY_GIT_COMMAND_MS,
+  TELEMETRY_GIT_COMMAND_ERRORS,
+} = require('../../ci-visibility/telemetry')
+const { storage } = require('../../../../datadog-core')
+const {
+  GIT_COMMIT_SHA,
+  GIT_BRANCH,
+  GIT_REPOSITORY_URL,
+  GIT_TAG,
+  GIT_COMMIT_MESSAGE,
+  GIT_COMMIT_COMMITTER_DATE,
+  GIT_COMMIT_COMMITTER_EMAIL,
+  GIT_COMMIT_COMMITTER_NAME,
+  GIT_COMMIT_AUTHOR_DATE,
+  GIT_COMMIT_AUTHOR_EMAIL,
+  GIT_COMMIT_AUTHOR_NAME,
+  CI_WORKSPACE_PATH,
+  GIT_COMMIT_HEAD_AUTHOR_DATE,
+  GIT_COMMIT_HEAD_AUTHOR_EMAIL,
+  GIT_COMMIT_HEAD_AUTHOR_NAME,
+  GIT_COMMIT_HEAD_COMMITTER_DATE,
+  GIT_COMMIT_HEAD_COMMITTER_EMAIL,
+  GIT_COMMIT_HEAD_COMMITTER_NAME,
+  GIT_COMMIT_HEAD_MESSAGE,
+} = require('./tags')
+const { filterSensitiveInfoFromRepository } = require('./url')
+const { cachedExec } = require('./git-cache')
+
+const GIT_REV_LIST_MAX_BUFFER = 12 * 1024 * 1024 // 12MB
+
+function sanitizedExec (
+  cmd,
+  flags,
+  operationMetric,
+  durationMetric,
+  errorMetric,
+  shouldTrim = true
+) {
+  return storage('legacy').run({ noop: true }, () => {
+    let startTime
+    if (operationMetric) {
+      incrementCountMetric(operationMetric.name, operationMetric.tags)
+    }
+    if (durationMetric) {
+      startTime = Date.now()
+    }
+    try {
+      let result = cachedExec(cmd, flags, { stdio: 'pipe' }).toString()
+
+      if (shouldTrim) {
+        result = result.replaceAll(/(\r\n|\n|\r)/gm, '')
+      }
+
+      if (durationMetric) {
+        distributionMetric(durationMetric.name, durationMetric.tags, Date.now() - startTime)
+      }
+      return result
+    } catch (err) {
+      if (errorMetric) {
+        incrementCountMetric(errorMetric.name, {
+          ...errorMetric.tags,
+          errorType: err.code,
+          exitCode: err.status || err.errno,
+        })
+      }
+      log.error('Git plugin error executing command', err)
+      return ''
+    }
+  })
+}
+
+function isDirectory (path) {
+  try {
+    const stats = fs.statSync(path)
+    return stats.isDirectory()
+  } catch {
+    return false
+  }
+}
+
+function isGitAvailable () {
+  const isWindows = os.platform() === 'win32'
+  const command = isWindows ? 'where' : 'which'
+  try {
+    cachedExec(command, ['git'])
+    return true
+  } catch {
+    incrementCountMetric(TELEMETRY_GIT_COMMAND_ERRORS, { command: 'check_git', exitCode: 'missing' })
+    return false
+  }
+}
+
+function isShallowRepository () {
+  return sanitizedExec(
+    'git',
+    ['rev-parse', '--is-shallow-repository'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'check_shallow' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'check_shallow' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'check_shallow' } }
+  ) === 'true'
+}
+
+function getGitVersion () {
+  const gitVersionString = sanitizedExec('git', ['version'])
+  const gitVersionMatches = /** @type {RegExpMatchArray} */ (gitVersionString.match(/git version (\d+)\.(\d+)\.(\d+)/))
+  try {
+    return {
+      major: Number.parseInt(gitVersionMatches[1]),
+      minor: Number.parseInt(gitVersionMatches[2]),
+      patch: Number.parseInt(gitVersionMatches[3]),
+    }
+  } catch {
+    return null
+  }
+}
+
+function unshallowRepository (parentOnly = false) {
+  const gitVersion = getGitVersion()
+  if (!gitVersion) {
+    log.warn('Git version could not be extracted, so git unshallow will not proceed')
+    return
+  }
+  if (gitVersion.major < 2 || (gitVersion.major === 2 && gitVersion.minor < 27)) {
+    log.warn('Git version is <2.27, so git unshallow will not proceed')
+    return
+  }
+  const defaultRemoteName = sanitizedExec('git', ['config', '--default', 'origin', '--get', 'clone.defaultRemoteName'])
+  const revParseHead = sanitizedExec('git', ['rev-parse', 'HEAD'])
+
+  const baseGitOptions = [
+    'fetch',
+    parentOnly ? '--deepen=1' : '--shallow-since="1 month ago"',
+    '--update-shallow',
+    '--filter=blob:none',
+    '--recurse-submodules=no',
+    defaultRemoteName,
+  ]
+
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'unshallow' })
+  const start = Date.now()
+  let flags = [
+    ...baseGitOptions,
+    revParseHead,
+  ]
+  try {
+    cachedExec('git', flags)
+  } catch (err) {
+    // If the local HEAD is a commit that has not been pushed to the remote, the above command will fail.
+    // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+    log.warn(() => `Git unshallow failed: ${flags.join(' ')}`)
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+    )
+    const upstreamRemote = sanitizedExec(
+      'git',
+      ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}']
+    )
+    flags = [
+      ...baseGitOptions,
+      upstreamRemote,
+    ]
+    try {
+      cachedExec('git', flags)
+    } catch (err) {
+      // If the CI is working on a detached HEAD or branch tracking hasn't been set up, the above command will fail.
+      // eslint-disable-next-line eslint-rules/eslint-log-printf-style
+      log.warn(() => `Git unshallow failed again: ${flags.join(' ')}`)
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'unshallow', errorType: err.code, exitCode: err.status || err.errno }
+      )
+      // We use sanitizedExec here because if this last option fails, we'll give up.
+      sanitizedExec(
+        'git',
+        baseGitOptions,
+        null,
+        null,
+        { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'unshallow' } } // we log the error in sanitizedExec
+      )
+    }
+  }
+  distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'unshallow' }, Date.now() - start)
+}
+
+function getRepositoryUrl () {
+  return sanitizedExec(
+    'git',
+    ['config', '--get', 'remote.origin.url'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_repository' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_repository' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_repository' } }
+  )
+}
+
+function getLatestCommits () {
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_local_commits' })
+  const startTime = Date.now()
+  try {
+    const result = cachedExec('git', ['log', '--format=%H', '-n 1000', '--since="1 month ago"'])
+      .toString()
+      .split('\n')
+      .filter(Boolean)
+    distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_local_commits' }, Date.now() - startTime)
+    return result
+  } catch (err) {
+    log.error('Get latest commits failed: %s', err.message)
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_local_commits', errorType: err.status }
+    )
+    return []
+  }
+}
+
+function getGitDiff (baseCommit, targetCommit) {
+  const flags = ['diff', '-U0', '--word-diff=porcelain', baseCommit]
+  if (targetCommit) {
+    flags.push(targetCommit)
+  }
+  return sanitizedExec(
+    'git',
+    flags,
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'diff' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'diff' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'diff' } },
+    false // important not to trim or we'll lose the line breaks which we need to detect impacted tests
+  )
+}
+
+function getGitRemoteName () {
+  const upstreamRemote = sanitizedExec(
+    'git',
+    ['rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_remote_name' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_remote_name' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_remote_name' } }
+  )
+
+  if (upstreamRemote) {
+    return upstreamRemote.split('/')[0]
+  }
+
+  const remotes = sanitizedExec(
+    'git',
+    ['remote'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_remote_name' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_remote_name' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_remote_name' } },
+    false
+  )
+
+  return remotes.split('\n')[0] || 'origin'
+}
+
+function getSourceBranch () {
+  return sanitizedExec(
+    'git',
+    ['rev-parse', '--abbrev-ref', 'HEAD'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_source_branch' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_source_branch' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_source_branch' } }
+  )
+}
+
+function checkAndFetchBranch (branch, remoteName) {
+  try {
+    // `git show-ref --verify --quiet refs/remotes/${remoteName}/${branch}` will exit 0 if the branch exists
+    // Otherwise it will exit 1
+    cachedExec(
+      'git',
+      ['show-ref', '--verify', '--quiet', `refs/remotes/${remoteName}/${branch}`],
+    )
+    // branch exists locally, so we finish
+  } catch {
+    // branch does not exist locally, so we will check the remote
+    try {
+      // IMPORTANT: we use timeouts because these commands hang if the branch can't be found
+      // `git ls-remote --heads origin my-branch` will exit 0 even if the branch doesn't exist.
+      // The piece of information we need is whether the command outputs anything.
+      // `git ls-remote --heads origin my-branch` could exit an error code if the remote does not exist.
+      const remoteHeads = cachedExec(
+        'git',
+        ['ls-remote', '--heads', remoteName, branch],
+        { stdio: 'pipe', timeout: 2000 }
+      )
+      if (remoteHeads) {
+        // branch exists, so we'll fetch it
+        cachedExec(
+          'git',
+          ['fetch', '--depth', '1', remoteName, branch],
+          { stdio: 'pipe', timeout: 5000 }
+        )
+      }
+    } catch (err) {
+      // branch does not exist or couldn't be fetched, so we can't do anything
+      log.debug('Git plugin error checking and fetching branch', err)
+    }
+  }
+}
+
+function getLocalBranches (remoteName) {
+  const localBranches = sanitizedExec(
+    'git',
+    ['for-each-ref', '--format=%(refname:short)', `refs/remotes/${remoteName}`],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_local_branches' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_local_branches' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_local_branches' } },
+    false
+  )
+  try {
+    return localBranches.split('\n').filter(Boolean)
+  } catch {
+    return []
+  }
+}
+
+function getMergeBase (baseBranch, sourceBranch) {
+  return sanitizedExec(
+    'git',
+    ['merge-base', baseBranch, sourceBranch],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_merge_base' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_merge_base' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_merge_base' } }
+  )
+}
+
+function getCounts (sourceBranch, candidateBranch) {
+  const counts = sanitizedExec(
+    'git',
+    ['rev-list', '--left-right', '--count', `${candidateBranch}...${sourceBranch}`],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_counts' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_counts' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_counts' } }
+  )
+  try {
+    if (!counts) {
+      return { behind: null, ahead: null }
+    }
+    const [behind, ahead] = counts.split(/\s+/).map(Number)
+    return { behind, ahead }
+  } catch {
+    return { behind: null, ahead: null }
+  }
+}
+
+function getCommitsRevList (commitsToExclude, commitsToInclude) {
+  let result = null
+
+  const commitsToExcludeString = commitsToExclude.map(commit => `^${commit}`)
+
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'get_objects' })
+  const startTime = Date.now()
+  try {
+    result = cachedExec(
+      'git',
+      [
+        'rev-list',
+        '--objects',
+        '--no-object-names',
+        '--filter=blob:none',
+        '--since="1 month ago"',
+        ...commitsToExcludeString,
+        ...commitsToInclude,
+      ],
+      { stdio: 'pipe', maxBuffer: GIT_REV_LIST_MAX_BUFFER })
+      .toString()
+      .split('\n')
+      .filter(Boolean)
+  } catch (err) {
+    log.error('Get commits to upload failed: %s', err.message)
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'get_objects', errorType: err.code, exitCode: err.status || err.errno } // err.status might be null
+    )
+  }
+  distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'get_objects' }, Date.now() - startTime)
+  return result
+}
+
+function generatePackFilesForCommits (commitsToUpload) {
+  let result = []
+  const tmpFolder = os.tmpdir()
+
+  if (!isDirectory(tmpFolder)) {
+    // TODO: Do we need the stack trace for this error? If not, just log the string
+    log.error(new Error('Provided path to generate packfiles is not a directory'))
+    return []
+  }
+
+  const randomPrefix = String(Math.floor(Math.random() * 10_000))
+  const temporaryPath = path.join(tmpFolder, randomPrefix)
+  const cwdPath = path.join(process.cwd(), randomPrefix)
+
+  incrementCountMetric(TELEMETRY_GIT_COMMAND, { command: 'pack_objects' })
+  const startTime = Date.now()
+  // Generates pack files to upload and
+  // returns the ordered list of packfiles' paths
+  function execGitPackObjects (targetPath) {
+    return cachedExec(
+      'git',
+      [
+        'pack-objects',
+        '--compression=9',
+        '--max-pack-size=3m',
+        targetPath,
+      ],
+      { stdio: 'pipe', input: commitsToUpload.join('\n') }
+    ).toString().split('\n').filter(Boolean).map(commit => `${targetPath}-${commit}.pack`)
+  }
+
+  try {
+    result = execGitPackObjects(temporaryPath)
+  } catch (err) {
+    log.error('Git plugin error executing git pack-objects command', err)
+    incrementCountMetric(
+      TELEMETRY_GIT_COMMAND_ERRORS,
+      { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+    )
+    /**
+     * The generation of pack files in the temporary folder (from `os.tmpdir()`)
+     * sometimes fails in certain CI setups with the error message
+     * `unable to rename temporary pack file: Invalid cross-device link`.
+     * The reason why is unclear.
+     *
+     * A workaround is to attempt to generate the pack files in `process.cwd()`.
+     * While this works most of the times, it's not ideal since it affects the git status.
+     * This workaround is intended to be temporary.
+     *
+     * TODO: fix issue and remove workaround.
+     */
+    try {
+      result = execGitPackObjects(cwdPath)
+    } catch (err) {
+      log.error('Git plugin error executing fallback git pack-objects command', err)
+      incrementCountMetric(
+        TELEMETRY_GIT_COMMAND_ERRORS,
+        { command: 'pack_objects', exitCode: err.status || err.errno, errorType: err.code }
+      )
+    }
+  }
+  distributionMetric(TELEMETRY_GIT_COMMAND_MS, { command: 'pack_objects' }, Date.now() - startTime)
+
+  return result
+}
+
+function getRepositoryRoot () {
+  return sanitizedExec(
+    'git',
+    ['rev-parse', '--show-toplevel']
+  )
+}
+
+// If there is ciMetadata, it takes precedence.
+function getGitMetadata (ciMetadata) {
+  const {
+    commitSHA,
+    branch,
+    repositoryUrl,
+    tag,
+    commitMessage,
+    authorName: ciAuthorName,
+    authorEmail: ciAuthorEmail,
+    ciWorkspacePath,
+    headCommitSha,
+  } = ciMetadata
+
+  // With stdio: 'pipe', errors in this command will not be output to the parent process,
+  // so if `git` is not present in the env, we won't show a warning to the user.
+  const [
+    authorName,
+    authorEmail,
+    authorDate,
+    committerName,
+    committerEmail,
+    committerDate,
+  ] = sanitizedExec('git', ['show', '-s', '--format=%an,%ae,%aI,%cn,%ce,%cI']).split(',')
+
+  const tags = {
+    [GIT_COMMIT_MESSAGE]:
+      commitMessage || sanitizedExec('git', ['show', '-s', '--format=%B'], null, null, null, false),
+    [GIT_BRANCH]: branch || sanitizedExec('git', ['rev-parse', '--abbrev-ref', 'HEAD']),
+    [GIT_COMMIT_SHA]: commitSHA || sanitizedExec('git', ['rev-parse', 'HEAD']),
+    [CI_WORKSPACE_PATH]: ciWorkspacePath || getRepositoryRoot(),
+  }
+
+  if (headCommitSha) {
+    if (isShallowRepository()) {
+      fetchHeadCommitSha(headCommitSha)
+    }
+
+    const [
+      gitHeadCommitSha,
+      headAuthorDate,
+      headAuthorName,
+      headAuthorEmail,
+      headCommitterDate,
+      headCommitterName,
+      headCommitterEmail,
+      headCommitMessage,
+    ] = sanitizedExec(
+      'git',
+      [
+        'show',
+        '-s',
+        '--format=\'%H","%aI","%an","%ae","%cI","%cn","%ce","%B\'',
+        headCommitSha,
+      ],
+      null,
+      null,
+      null,
+      false
+    ).split('","')
+
+    if (gitHeadCommitSha) {
+      tags[GIT_COMMIT_HEAD_AUTHOR_DATE] = headAuthorDate
+      tags[GIT_COMMIT_HEAD_AUTHOR_EMAIL] = headAuthorEmail
+      tags[GIT_COMMIT_HEAD_AUTHOR_NAME] = headAuthorName
+      tags[GIT_COMMIT_HEAD_COMMITTER_DATE] = headCommitterDate
+      tags[GIT_COMMIT_HEAD_COMMITTER_EMAIL] = headCommitterEmail
+      tags[GIT_COMMIT_HEAD_COMMITTER_NAME] = headCommitterName
+      tags[GIT_COMMIT_HEAD_MESSAGE] = headCommitMessage
+    }
+  }
+
+  const entries = [
+    GIT_REPOSITORY_URL,
+    filterSensitiveInfoFromRepository(repositoryUrl || sanitizedExec('git', ['ls-remote', '--get-url'])),
+    GIT_COMMIT_AUTHOR_DATE, authorDate,
+    GIT_COMMIT_AUTHOR_NAME, ciAuthorName || authorName,
+    GIT_COMMIT_AUTHOR_EMAIL, ciAuthorEmail || authorEmail,
+    GIT_COMMIT_COMMITTER_DATE, committerDate,
+    GIT_COMMIT_COMMITTER_NAME, committerName,
+    GIT_COMMIT_COMMITTER_EMAIL, committerEmail,
+    GIT_TAG, tag,
+  ]
+
+  for (let i = 0; i < entries.length; i += 2) {
+    const value = entries[i + 1]
+    if (value) {
+      tags[entries[i]] = value
+    }
+  }
+
+  return tags
+}
+
+function getGitInformationDiscrepancy () {
+  const gitRepositoryUrl = getRepositoryUrl()
+
+  const gitCommitSHA = sanitizedExec(
+    'git',
+    ['rev-parse', 'HEAD'],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'get_commit_sha' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'get_commit_sha' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'get_commit_sha' } }
+  )
+
+  return { gitRepositoryUrl, gitCommitSHA }
+}
+
+function fetchHeadCommitSha (headSha) {
+  const remoteName = getGitRemoteName()
+
+  sanitizedExec(
+    'git',
+    [
+      'fetch',
+      '--update-shallow',
+      '--filter=blob:none',
+      '--recurse-submodules=no',
+      '--no-write-fetch-head',
+      remoteName,
+      headSha,
+    ],
+    { name: TELEMETRY_GIT_COMMAND, tags: { command: 'fetch_head_commit_sha' } },
+    { name: TELEMETRY_GIT_COMMAND_MS, tags: { command: 'fetch_head_commit_sha' } },
+    { name: TELEMETRY_GIT_COMMAND_ERRORS, tags: { command: 'fetch_head_commit_sha' } }
+  )
+}
+
+module.exports = {
+  getGitMetadata,
+  getLatestCommits,
+  getRepositoryUrl,
+  generatePackFilesForCommits,
+  getCommitsRevList,
+  GIT_REV_LIST_MAX_BUFFER,
+  isShallowRepository,
+  unshallowRepository,
+  isGitAvailable,
+  getGitInformationDiscrepancy,
+  getGitDiff,
+  getGitRemoteName,
+  getSourceBranch,
+  checkAndFetchBranch,
+  getLocalBranches,
+  getMergeBase,
+  getCounts,
+  fetchHeadCommitSha,
+  getRepositoryRoot,
+}