datadog-ci 1.0.0.beta1 → 1.0.0.beta2

data/lib/datadog/ci/ext/environment.rb

@@ -3,6 +3,8 @@
 require_relative "git"
 require_relative "environment/extractor"
 
+require_relative "../utils/git"
+
 module Datadog
   module CI
     module Ext
@@ -21,8 +23,6 @@ module Datadog
         TAG_NODE_NAME = "ci.node.name"
         TAG_CI_ENV_VARS = "_dd.ci.env_vars"
 
-        HEX_NUMBER_REGEXP = /[0-9a-f]{40}/i.freeze
-
         module_function
 
         def tags(env)
@@ -57,24 +57,19 @@ module Datadog
         end
 
         def validate_git_sha(git_sha)
-          message = "DD_GIT_COMMIT_SHA must be a full-length git SHA."
+          return if Utils::Git.valid_commit_sha?(git_sha)
 
-          if git_sha.nil? || git_sha.empty?
-            message += " No value was set and no SHA was automatically extracted."
-            Datadog.logger.error(message)
-            return
-          end
+          message = "DD_GIT_COMMIT_SHA must be a full-length git SHA."
 
-          if git_sha.length < Git::SHA_LENGTH
-            message += " Expected SHA length #{Git::SHA_LENGTH}, was #{git_sha.length}."
-            Datadog.logger.error(message)
-            return
+          message += if git_sha.nil? || git_sha.empty?
+            " No value was set and no SHA was automatically extracted."
+          elsif git_sha.length < Git::SHA_LENGTH
+            " Expected SHA length #{Git::SHA_LENGTH}, was #{git_sha.length}."
+          else
+            " Expected SHA to be a valid HEX number, got #{git_sha}."
           end
 
-          unless HEX_NUMBER_REGEXP.match?(git_sha)
-            message += " Expected SHA to be a valid HEX number, got #{git_sha}."
-            Datadog.logger.error(message)
-          end
+          Datadog.logger.error(message)
         end
       end
     end

data/lib/datadog/ci/ext/settings.rb

@@ -11,6 +11,7 @@ module Datadog
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED"
         ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY"
         ENV_ITR_ENABLED = "DD_CIVISIBILITY_ITR_ENABLED"
+        ENV_GIT_METADATA_UPLOAD_ENABLED = "DD_CIVISIBILITY_GIT_METADATA_UPLOAD_ENABLED"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = [

data/lib/datadog/ci/ext/test.rb

@@ -26,6 +26,9 @@ module Datadog
         # ITR tags
         TAG_ITR_TEST_SKIPPING_ENABLED = "test.itr.tests_skipping.enabled"
         TAG_ITR_TEST_SKIPPING_TYPE = "test.itr.tests_skipping.type"
+        TAG_ITR_TEST_SKIPPING_COUNT = "test.itr.tests_skipping.count"
+        TAG_ITR_SKIPPED_BY_ITR = "test.skipped_by_itr"
+        TAG_ITR_TESTS_SKIPPED = "_dd.ci.itr.tests_skipped"
 
         # Code coverage tags
         TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
@@ -43,6 +46,7 @@ module Datadog
         # Environment runtime tags
         TAG_OS_ARCHITECTURE = "os.architecture"
         TAG_OS_PLATFORM = "os.platform"
+        TAG_OS_VERSION = "os.version"
         TAG_RUNTIME_NAME = "runtime.name"
         TAG_RUNTIME_VERSION = "runtime.version"
 
@@ -53,6 +57,7 @@ module Datadog
         # could be either "test" or "suite" depending on whether we skip individual tests or whole suites
         # we use test skipping for Ruby
         ITR_TEST_SKIPPING_MODE = "test"
+        ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
 
         # test status as recognized by Datadog
         module Status

data/lib/datadog/ci/ext/transport.rb

@@ -27,6 +27,7 @@ module Datadog
         TEST_COVERAGE_INTAKE_PATH = "/api/v2/citestcov"
 
         DD_API_HOST_PREFIX = "api"
+
         DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting"
         DD_API_SETTINGS_TYPE = "ci_app_test_service_libraries_settings"
         DD_API_SETTINGS_RESPONSE_DIG_KEYS = %w[data attributes].freeze
@@ -36,6 +37,13 @@ module Datadog
         DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
         DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
 
+        DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"
+
+        DD_API_GIT_UPLOAD_PACKFILE_PATH = "/api/v2/git/repository/packfile"
+
+        DD_API_SKIPPABLE_TESTS_PATH = "/api/v2/ci/tests/skippable"
+        DD_API_SKIPPABLE_TESTS_TYPE = "test_params"
+
         CONTENT_TYPE_MESSAGEPACK = "application/msgpack"
         CONTENT_TYPE_JSON = "application/json"
         CONTENT_TYPE_MULTIPART_FORM_DATA = "multipart/form-data"

data/lib/datadog/ci/git/local_repository.rb

@@ -0,0 +1,238 @@
+# frozen_string_literal: true
+
+require "open3"
+require "pathname"
+
+require_relative "user"
+
+module Datadog
+  module CI
+    module Git
+      module LocalRepository
+        COMMAND_RETRY_COUNT = 3
+
+        def self.root
+          return @root if defined?(@root)
+
+          @root = git_root || Dir.pwd
+        end
+
+        def self.relative_to_root(path)
+          return "" if path.nil?
+
+          root_path = root
+          return path if root_path.nil?
+
+          path = Pathname.new(File.expand_path(path))
+          root_path = Pathname.new(root_path)
+
+          path.relative_path_from(root_path).to_s
+        end
+
+        def self.repository_name
+          return @repository_name if defined?(@repository_name)
+
+          git_remote_url = git_repository_url
+
+          # return git repository name from remote url without .git extension
+          last_path_segment = git_remote_url.split("/").last if git_remote_url
+          @repository_name = last_path_segment.gsub(".git", "") if last_path_segment
+          @repository_name ||= current_folder_name
+        rescue => e
+          log_failure(e, "git repository name")
+          @repository_name = current_folder_name
+        end
+
+        def self.current_folder_name
+          File.basename(root)
+        end
+
+        def self.git_repository_url
+          exec_git_command("git ls-remote --get-url")
+        rescue => e
+          log_failure(e, "git repository url")
+          nil
+        end
+
+        def self.git_root
+          exec_git_command("git rev-parse --show-toplevel")
+        rescue => e
+          log_failure(e, "git root path")
+          nil
+        end
+
+        def self.git_commit_sha
+          exec_git_command("git rev-parse HEAD")
+        rescue => e
+          log_failure(e, "git commit sha")
+          nil
+        end
+
+        def self.git_branch
+          exec_git_command("git rev-parse --abbrev-ref HEAD")
+        rescue => e
+          log_failure(e, "git branch")
+          nil
+        end
+
+        def self.git_tag
+          exec_git_command("git tag --points-at HEAD")
+        rescue => e
+          log_failure(e, "git tag")
+          nil
+        end
+
+        def self.git_commit_message
+          exec_git_command("git show -s --format=%s")
+        rescue => e
+          log_failure(e, "git commit message")
+          nil
+        end
+
+        def self.git_commit_users
+          # Get committer and author information in one command.
+          output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
+          unless output
+            Datadog.logger.debug(
+              "Unable to read git commit users: git command output is nil"
+            )
+            nil_user = NilUser.new
+            return [nil_user, nil_user]
+          end
+
+          author_name, author_email, author_timestamp,
+            committer_name, committer_email, committer_timestamp = output.split("\t").each(&:strip!)
+
+          author = User.new(author_name, author_email, author_timestamp)
+          committer = User.new(committer_name, committer_email, committer_timestamp)
+
+          [author, committer]
+        rescue => e
+          log_failure(e, "git commit users")
+
+          nil_user = NilUser.new
+          [nil_user, nil_user]
+        end
+
+        # returns maximum of 1000 latest commits in the last month
+        def self.git_commits
+          output = exec_git_command("git log --format=%H -n 1000 --since=\"1 month ago\"")
+          return [] if output.nil?
+
+          output.split("\n")
+        rescue => e
+          log_failure(e, "git commits")
+          []
+        end
+
+        def self.git_commits_rev_list(included_commits:, excluded_commits:)
+          included_commits = filter_invalid_commits(included_commits).join(" ")
+          excluded_commits = filter_invalid_commits(excluded_commits).map! { |sha| "^#{sha}" }.join(" ")
+
+          exec_git_command(
+            "git rev-list " \
+            "--objects " \
+            "--no-object-names " \
+            "--filter=blob:none " \
+            "--since=\"1 month ago\" " \
+            "#{excluded_commits} #{included_commits}"
+          )
+        rescue => e
+          log_failure(e, "git commits rev list")
+          nil
+        end
+
+        def self.git_generate_packfiles(included_commits:, excluded_commits:, path:)
+          return nil unless File.exist?(path)
+
+          commit_tree = git_commits_rev_list(included_commits: included_commits, excluded_commits: excluded_commits)
+          return nil if commit_tree.nil?
+
+          basename = SecureRandom.hex(4)
+
+          exec_git_command(
+            "git pack-objects --compression=9 --max-pack-size=3m #{path}/#{basename}",
+            stdin: commit_tree
+          )
+
+          basename
+        rescue => e
+          log_failure(e, "git generate packfiles")
+          nil
+        end
+
+        def self.git_shallow_clone?
+          exec_git_command("git rev-parse --is-shallow-repository") == "true"
+        rescue => e
+          log_failure(e, "git shallow clone")
+          false
+        end
+
+        def self.git_unshallow
+          exec_git_command(
+            "git fetch " \
+            "--shallow-since=\"1 month ago\" " \
+            "--update-shallow " \
+            "--filter=\"blob:none\" " \
+            "--recurse-submodules=no " \
+            "$(git config --default origin --get clone.defaultRemoteName) $(git rev-parse HEAD)"
+          )
+        rescue => e
+          log_failure(e, "git unshallow")
+          nil
+        end
+
+        # makes .exec_git_command private to make sure that this method
+        # is not called from outside of this module with insecure parameters
+        class << self
+          private
+
+          def filter_invalid_commits(commits)
+            commits.filter { |commit| Utils::Git.valid_commit_sha?(commit) }
+          end
+
+          def exec_git_command(cmd, stdin: nil)
+            # Shell injection is alleviated by making sure that no outside modules call this method.
+            # It is called only internally with static parameters.
+            # no-dd-sa:ruby-security/shell-injection
+            out, status = Open3.capture2e(cmd, stdin_data: stdin)
+
+            if status.nil?
+              retry_count = COMMAND_RETRY_COUNT
+              Datadog.logger.debug { "Opening pipe failed, starting retries..." }
+              while status.nil? && retry_count.positive?
+                # no-dd-sa:ruby-security/shell-injection
+                out, status = Open3.capture2e(cmd, stdin_data: stdin)
+                Datadog.logger.debug { "After retry status is [#{status}]" }
+                retry_count -= 1
+              end
+            end
+
+            if status.nil? || !status.success?
+              raise "Failed to run git command [#{cmd}] with input [#{stdin}] and output [#{out}]"
+            end
+
+            # Sometimes Encoding.default_external is somehow set to US-ASCII which breaks
+            # commit messages with UTF-8 characters like emojis
+            # We force output's encoding to be UTF-8 in this case
+            # This is safe to do as UTF-8 is compatible with US-ASCII
+            if Encoding.default_external == Encoding::US_ASCII
+              out = out.force_encoding(Encoding::UTF_8)
+            end
+            out.strip! # There's always a "\n" at the end of the command output
+
+            return nil if out.empty?
+
+            out
+          end
+
+          def log_failure(e, action)
+            Datadog.logger.debug(
+              "Unable to perform #{action}: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/packfiles.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+
+require_relative "local_repository"
+
+module Datadog
+  module CI
+    module Git
+      module Packfiles
+        def self.generate(included_commits:, excluded_commits:)
+          # @type var current_process_tmp_folder: String?
+          current_process_tmp_folder = nil
+
+          Dir.mktmpdir do |tmpdir|
+            prefix = LocalRepository.git_generate_packfiles(
+              included_commits: included_commits,
+              excluded_commits: excluded_commits,
+              path: tmpdir
+            )
+
+            if prefix.nil?
+              # git pack-files command fails if tmpdir is mounted on
+              # a different device from the current process directory
+              #
+              # @type var current_process_tmp_folder: String
+              current_process_tmp_folder = File.join(Dir.pwd, "tmp", "packfiles")
+              FileUtils.mkdir_p(current_process_tmp_folder)
+
+              prefix = LocalRepository.git_generate_packfiles(
+                included_commits: included_commits,
+                excluded_commits: excluded_commits,
+                path: current_process_tmp_folder
+              )
+
+              if prefix.nil?
+                Datadog.logger.debug("Packfiles generation failed twice, aborting")
+                break
+              end
+
+              tmpdir = current_process_tmp_folder
+            end
+
+            packfiles = Dir.entries(tmpdir) - %w[. ..]
+            if packfiles.empty?
+              Datadog.logger.debug("Empty packfiles list, aborting process")
+              break
+            end
+
+            packfiles.each do |packfile_name|
+              next unless packfile_name.start_with?(prefix)
+              next unless packfile_name.end_with?(".pack")
+
+              packfile_path = File.join(tmpdir, packfile_name)
+
+              yield packfile_path
+            end
+          end
+        rescue => e
+          Datadog.logger.debug("Packfiles could not be generated, error: #{e}")
+        ensure
+          if current_process_tmp_folder && File.exist?(current_process_tmp_folder)
+            FileUtils.remove_entry(current_process_tmp_folder)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/search_commits.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+
+require_relative "../ext/transport"
+require_relative "../utils/git"
+
+module Datadog
+  module CI
+    module Git
+      class SearchCommits
+        class ApiError < StandardError; end
+
+        attr_reader :api
+
+        def initialize(api:)
+          @api = api
+        end
+
+        def call(repository_url, commits)
+          raise ApiError, "test visibility API is not configured" if api.nil?
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_GIT_SEARCH_COMMITS_PATH,
+            payload: request_payload(repository_url, commits)
+          )
+          raise ApiError, "Failed to search commits: #{http_response.inspect}" unless http_response.ok?
+
+          response_payload = parse_json_response(http_response)
+          extract_commits(response_payload)
+        end
+
+        private
+
+        def request_payload(repository_url, commits)
+          {
+            meta: {
+              repository_url: repository_url
+            },
+            data: commits.filter_map do |commit|
+              next unless Utils::Git.valid_commit_sha?(commit)
+
+              {
+                id: commit,
+                type: "commit"
+              }
+            end
+          }.to_json
+        end
+
+        def parse_json_response(http_response)
+          JSON.parse(http_response.payload)
+        rescue JSON::ParserError => e
+          raise ApiError, "Failed to parse search commits response: #{e}. Payload was: #{http_response.payload}"
+        end
+
+        def extract_commits(response_payload)
+          result = Set.new
+
+          response_payload.fetch("data").each do |commit_json|
+            raise ApiError, "Invalid commit type response #{commit_json}" unless commit_json["type"] == "commit"
+
+            commit_sha = commit_json["id"]
+            raise ApiError, "Invalid commit SHA response #{commit_sha}" unless Utils::Git.valid_commit_sha?(commit_sha)
+
+            result.add(commit_sha)
+          end
+
+          result
+        rescue KeyError => e
+          raise ApiError, "Malformed search commits response: #{e}. Payload was: #{response_payload}"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/tree_uploader.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "fileutils"
+
+require_relative "local_repository"
+require_relative "search_commits"
+require_relative "upload_packfile"
+require_relative "packfiles"
+
+module Datadog
+  module CI
+    module Git
+      class TreeUploader
+        attr_reader :api
+
+        def initialize(api:)
+          @api = api
+        end
+
+        def call(repository_url)
+          if api.nil?
+            Datadog.logger.debug("API is not configured, aborting git upload")
+            return
+          end
+
+          Datadog.logger.debug { "Uploading git tree for repository #{repository_url}" }
+
+          latest_commits = LocalRepository.git_commits
+          head_commit = latest_commits&.first
+          if head_commit.nil?
+            Datadog.logger.debug("Got empty latest commits list, aborting git upload")
+            return
+          end
+
+          begin
+            # ask the backend for the list of commits it already has
+            known_commits, new_commits = fetch_known_commits_and_split(repository_url, latest_commits)
+            # if all commits are present in the backend, we don't need to upload anything
+            if new_commits.empty?
+              Datadog.logger.debug("No new commits to upload")
+              return
+            end
+
+            # quite often we deal with shallow clones in CI environment
+            if LocalRepository.git_shallow_clone? && LocalRepository.git_unshallow
+              Datadog.logger.debug("Detected shallow clone and unshallowed the repository, repeating commits search")
+
+              # re-run the search with the updated commit list after unshallowing
+              known_commits, new_commits = fetch_known_commits_and_split(
+                repository_url,
+                LocalRepository.git_commits
+              )
+            end
+          rescue SearchCommits::ApiError => e
+            Datadog.logger.debug("SearchCommits failed with #{e}, aborting git upload")
+            return
+          end
+
+          Datadog.logger.debug { "Uploading packfiles for commits: #{new_commits}" }
+          uploader = UploadPackfile.new(
+            api: api,
+            head_commit_sha: head_commit,
+            repository_url: repository_url
+          )
+          Packfiles.generate(included_commits: new_commits, excluded_commits: known_commits) do |filepath|
+            uploader.call(filepath: filepath)
+          rescue UploadPackfile::ApiError => e
+            Datadog.logger.debug("Packfile upload failed with #{e}")
+            break
+          end
+        ensure
+          Datadog.logger.debug("Git tree upload finished")
+        end
+
+        private
+
+        # Split the latest commits list into known and new commits
+        # based on the backend response provided by /search_commits endpoint
+        def fetch_known_commits_and_split(repository_url, latest_commits)
+          Datadog.logger.debug { "Checking the latest commits list with backend: #{latest_commits}" }
+          backend_commits = SearchCommits.new(api: api).call(repository_url, latest_commits)
+          latest_commits.partition do |commit|
+            backend_commits.include?(commit)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/upload_packfile.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+
+module Datadog
+  module CI
+    module Git
+      class UploadPackfile
+        class ApiError < StandardError; end
+
+        attr_reader :api, :head_commit_sha, :repository_url
+
+        def initialize(api:, head_commit_sha:, repository_url:)
+          @api = api
+          @head_commit_sha = head_commit_sha
+          @repository_url = repository_url
+        end
+
+        def call(filepath:)
+          raise ApiError, "test visibility API is not configured" if api.nil?
+
+          payload_boundary = SecureRandom.uuid
+
+          filename = File.basename(filepath)
+          packfile_contents = read_file(filepath)
+
+          payload = request_payload(payload_boundary, filename, packfile_contents)
+          content_type = "#{Ext::Transport::CONTENT_TYPE_MULTIPART_FORM_DATA}; boundary=#{payload_boundary}"
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_GIT_UPLOAD_PACKFILE_PATH,
+            payload: payload,
+            headers: {Ext::Transport::HEADER_CONTENT_TYPE => content_type}
+          )
+
+          raise ApiError, "Failed to upload packfile: #{http_response.inspect}" unless http_response.ok?
+        end
+
+        private
+
+        def request_payload(boundary, filename, packfile_contents)
+          [
+            "--#{boundary}",
+            'Content-Disposition: form-data; name="pushedSha"',
+            "Content-Type: application/json",
+            "",
+            {data: {id: head_commit_sha, type: "commit"}, meta: {repository_url: repository_url}}.to_json,
+            "--#{boundary}",
+            "Content-Disposition: form-data; name=\"packfile\"; filename=\"#{filename}\"",
+            "Content-Type: application/octet-stream",
+            "",
+            packfile_contents,
+            "--#{boundary}--"
+          ].join("\r\n")
+        end
+
+        def read_file(filepath)
+          File.read(filepath)
+        rescue => e
+          raise ApiError, "Failed to read packfile: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/git/user.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Git
+      class User
+        attr_reader :name, :email, :timestamp
+
+        def initialize(name, email, timestamp)
+          @name = name
+          @email = email
+          @timestamp = timestamp
+        end
+
+        def date
+          return nil if timestamp.nil?
+
+          Time.at(timestamp.to_i).utc.to_datetime.iso8601
+        end
+      end
+
+      class NilUser < User
+        def initialize
+          super(nil, nil, nil)
+        end
+      end
+    end
+  end
+end