datadog-ci 1.15.0 → 1.17.0

data/lib/datadog/ci/contrib/lograge/log_subscriber.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        module LogSubscriber
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            private
+
+            def before_format(data, payload)
+              return super unless datadog_logs_component.enabled
+              return super unless datadog_configuration[:enabled]
+
+              result = super
+
+              if result.fetch(:dd, {}).fetch(:trace_id, nil).nil?
+                Datadog.logger.debug { "Discarding uncorrelated log event: #{result.inspect}" }
+                return result
+              end
+              datadog_logs_component.write(result)
+              result
+            end
+
+            def datadog_logs_component
+              Datadog.send(:components).agentless_logs_submission
+            end
+
+            def datadog_configuration
+              Datadog.configuration.ci[:lograge]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/lograge/patcher.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "../patcher"
+require_relative "log_subscriber"
+
+module Datadog
+  module CI
+    module Contrib
+      module Lograge
+        # Patcher enables patching of lograge module
+        module Patcher
+          include Datadog::CI::Contrib::Patcher
+
+          module_function
+
+          def patch
+            unless datadog_logs_component.enabled
+              Datadog.logger.debug("Datadog logs submission is disabled, skipping lograge patching")
+              return
+            end
+
+            ::Lograge::LogSubscribers::Base.include(LogSubscriber)
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/minitest/runner.rb

@@ -38,6 +38,7 @@ module Datadog
             def run_one_method(klass, method_name)
               return super unless datadog_configuration[:enabled]
 
+              # @type var result: untyped
               result = nil
 
               test_retries_component.with_retries do

data/lib/datadog/ci/contrib/minitest/test.rb

@@ -40,7 +40,11 @@ module Datadog
                 },
                 service: datadog_configuration[:service_name]
               )
+              # Steep type checker doesn't know that we patched Minitest::Test class definition
+              #
+              # steep:ignore:start
               test_span&.itr_unskippable! if self.class.dd_suite_unskippable? || self.class.dd_test_unskippable?(name)
+              # steep:ignore:end
               skip(test_span&.datadog_skip_reason) if test_span&.should_skip?
             end
 
@@ -100,9 +104,10 @@ module Datadog
             end
 
             def dd_test_unskippable?(test_name)
-              return false unless @datadog_itr_unskippable_tests
+              tests = @datadog_itr_unskippable_tests
+              return false unless tests
 
-              @datadog_itr_unskippable_tests.include?(test_name)
+              tests.include?(test_name)
             end
           end
         end

data/lib/datadog/ci/contrib/parallel_tests/patcher.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require_relative "cli"
-require_relative "integration"
 require_relative "../patcher"
 
 module Datadog
@@ -12,9 +11,7 @@ module Datadog
         module Patcher
           include Datadog::CI::Contrib::Patcher
 
-          module_function
-
-          def patch
+          def self.patch
             ::ParallelTests::CLI.include(CLI)
           end
         end

data/lib/datadog/ci/contrib/patcher.rb

@@ -29,9 +29,13 @@ module Datadog
             return unless defined?(super)
 
             patch_only_once.run do
+              # There is no way to tell Steep that we prepend these methods to modules that have .patch method
+              #
+              # steep:ignore:start
               super.tap do
                 @patch_successful = true
               end
+              # steep:ignore:end
             rescue => e
               on_patch_error(e)
             end

data/lib/datadog/ci/contrib/rspec/helpers.rb

@@ -6,9 +6,7 @@ module Datadog
       module RSpec
         # Helper methods for RSpec instrumentation
         module Helpers
-          module_function
-
-          def parallel_tests?
+          def self.parallel_tests?
             !!ENV.fetch("TEST_ENV_NUMBER", nil) && !!ENV.fetch("PARALLEL_TEST_GROUPS", nil)
           end
         end

data/lib/datadog/ci/contrib/semantic_logger/configuration/settings.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "../ext"
+require_relative "../../settings"
+require_relative "../../../utils/configuration"
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        module Configuration
+          # Custom settings for the SemanticLogger integration
+          # @public_api
+          class Settings < Datadog::CI::Contrib::Settings
+            option :enabled do |o|
+              o.type :bool
+              o.env Ext::ENV_ENABLED
+              o.default true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/ext.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        # Datadog SemanticLogger integration constants
+        module Ext
+          ENV_ENABLED = "DD_CI_SEMANTIC_LOGGER_ENABLED"
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/integration.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+require_relative "configuration/settings"
+require_relative "patcher"
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        # Description of SemanticLogger integration
+        class Integration < Datadog::CI::Contrib::Integration
+          MINIMUM_VERSION = Gem::Version.new("4.0")
+
+          def version
+            Gem.loaded_specs["semantic_logger"]&.version
+          end
+
+          def loaded?
+            !defined?(::SemanticLogger).nil? && !defined?(::SemanticLogger::Logger).nil?
+          end
+
+          def compatible?
+            super && version >= MINIMUM_VERSION
+          end
+
+          def late_instrument?
+            true
+          end
+
+          def new_configuration
+            Configuration::Settings.new
+          end
+
+          def patcher
+            Patcher
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/logger.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        module Logger
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          module InstanceMethods
+            def log(log, message = nil, progname = nil, &block)
+              return super unless log.is_a?(::SemanticLogger::Log)
+              return super unless datadog_logs_component.enabled
+
+              result = super
+
+              datadog_logs_component.write(log.to_h.clone)
+
+              result
+            end
+
+            def datadog_logs_component
+              Datadog.send(:components).agentless_logs_submission
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/contrib/semantic_logger/patcher.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "../patcher"
+require_relative "logger"
+
+module Datadog
+  module CI
+    module Contrib
+      module SemanticLogger
+        # Patcher enables patching of semantic_logger module
+        module Patcher
+          include Datadog::CI::Contrib::Patcher
+
+          module_function
+
+          def patch
+            unless datadog_logs_component.enabled
+              Datadog.logger.debug("Datadog logs submission is disabled, skipping semantic_logger patching")
+              return
+            end
+
+            ::SemanticLogger::Logger.include(Logger)
+          end
+
+          def datadog_logs_component
+            Datadog.send(:components).agentless_logs_submission
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/ext/environment/extractor.rb

@@ -46,14 +46,12 @@ module Datadog
               Git::TAG_COMMIT_COMMITTER_EMAIL => @provider.git_commit_committer_email,
               Git::TAG_COMMIT_COMMITTER_NAME => @provider.git_commit_committer_name,
               Git::TAG_COMMIT_MESSAGE => @provider.git_commit_message,
-              Git::TAG_COMMIT_SHA => @provider.git_commit_sha
+              Git::TAG_COMMIT_SHA => @provider.git_commit_sha,
+              Git::TAG_PULL_REQUEST_BASE_BRANCH => @provider.git_pull_request_base_branch,
+              Git::TAG_PULL_REQUEST_BASE_BRANCH_SHA => @provider.git_pull_request_base_branch_sha,
+              Git::TAG_COMMIT_HEAD_SHA => @provider.git_commit_head_sha
             }
 
-            # set additional tags if provider needs them
-            @provider.additional_tags.each do |key, value|
-              @tags[key] = value
-            end
-
             # Normalize Git references and filter sensitive data
             normalize_git!
             # Expand ~

data/lib/datadog/ci/ext/environment/providers/appveyor.rb

@@ -83,6 +83,11 @@ module Datadog
               commit_message
             end
 
+            def git_pull_request_base_branch
+              # from docs: build branch. For Pull Request commits it is base branch PR is merging into
+              env["APPVEYOR_REPO_BRANCH"]
+            end
+
             private
 
             def github_repo_provider?

data/lib/datadog/ci/ext/environment/providers/base.rb

@@ -96,8 +96,13 @@ module Datadog
             def git_commit_sha
             end
 
-            def additional_tags
-              {}
+            def git_pull_request_base_branch
+            end
+
+            def git_pull_request_base_branch_sha
+            end
+
+            def git_commit_head_sha
             end
 
             private

data/lib/datadog/ci/ext/environment/providers/bitbucket.rb

@@ -59,6 +59,12 @@ module Datadog
               env["BITBUCKET_TAG"]
             end
 
+            def git_pull_request_base_branch
+              # from docs: The pull request destination branch (used in combination with BITBUCKET_BRANCH).
+              # Only available on a pull request triggered build
+              env["BITBUCKET_PR_DESTINATION_BRANCH"]
+            end
+
             private
 
             def url

data/lib/datadog/ci/ext/environment/providers/bitrise.rb

@@ -47,7 +47,7 @@ module Datadog
             end
 
             def git_branch
-              env["BITRISEIO_GIT_BRANCH_DEST"] || env["BITRISE_GIT_BRANCH"]
+              env["BITRISEIO_PULL_REQUEST_HEAD_BRANCH"] || env["BITRISE_GIT_BRANCH"]
             end
 
             def git_tag
@@ -73,6 +73,12 @@ module Datadog
             def git_commit_committer_email
               env["GIT_CLONE_COMMIT_COMMITER_EMAIL"] || env["GIT_CLONE_COMMIT_COMMITER_NAME"]
             end
+
+            def git_pull_request_base_branch
+              # from docs: Used only with builds triggered by pull requests: the destination/target branch of the pull request that triggered the build.
+              # For example, a pull request wants to merge the content of a branch into the branch main. In this case, this Env Var’s value is main.
+              env["BITRISEIO_GIT_BRANCH_DEST"]
+            end
           end
         end
       end

data/lib/datadog/ci/ext/environment/providers/buddy.rb

@@ -65,6 +65,11 @@ module Datadog
             def git_commit_committer_email
               env["BUDDY_EXECUTION_REVISION_COMMITTER_EMAIL"]
             end
+
+            def git_pull_request_base_branch
+              # from docs: The name of the Git BASE branch of the currently run Pull Request
+              env["BUDDY_RUN_PR_BASE_BRANCH"]
+            end
           end
         end
       end

data/lib/datadog/ci/ext/environment/providers/github_actions.rb

@@ -76,34 +76,53 @@ module Datadog
               }.reject { |_, v| v.nil? }.to_json
             end
 
-            def additional_tags
-              base_ref = env["GITHUB_BASE_REF"]
-              return {} if base_ref.nil? || base_ref.empty?
+            def git_pull_request_base_branch
+              return nil if github_event_json.nil?
 
-              # @type var result: Hash[String, String]
-              result = {
-                Git::TAG_PULL_REQUEST_BASE_BRANCH => base_ref
-              }
-
-              event_path = env["GITHUB_EVENT_PATH"]
-              event_json = JSON.parse(File.read(event_path))
+              env["GITHUB_BASE_REF"]
+            end
 
-              head_sha = event_json.dig("pull_request", "head", "sha")
-              result[Git::TAG_COMMIT_HEAD_SHA] = head_sha if head_sha
+            def git_pull_request_base_branch_sha
+              return nil if git_pull_request_base_branch.nil?
 
-              base_sha = event_json.dig("pull_request", "base", "sha")
-              result[Git::TAG_PULL_REQUEST_BASE_BRANCH_SHA] = base_sha if base_sha
+              event_json = github_event_json
+              return nil if event_json.nil?
 
-              result
+              event_json.dig("pull_request", "base", "sha")
             rescue => e
-              Datadog.logger.error("Failed to extract additional tags from GitHub Actions: #{e}")
-              Core::Telemetry::Logger.report(e, description: "Failed to extract additional tags from GitHub Actions")
+              Datadog.logger.error("Failed to extract pull request base branch SHA from GitHub Actions: #{e}")
+              Core::Telemetry::Logger.report(e, description: "Failed to extract pull request base branch SHA from GitHub Actions")
+              nil
+            end
 
-              {}
+            def git_commit_head_sha
+              return nil if git_pull_request_base_branch.nil?
+
+              event_json = github_event_json
+              return nil if event_json.nil?
+
+              event_json.dig("pull_request", "head", "sha")
+            rescue => e
+              Datadog.logger.error("Failed to extract commit head SHA from GitHub Actions: #{e}")
+              Core::Telemetry::Logger.report(e, description: "Failed to extract commit head SHA from GitHub Actions")
+              nil
             end
 
             private
 
+            def github_event_json
+              return @github_event_json if defined?(@github_event_json)
+
+              event_path = env["GITHUB_EVENT_PATH"]
+              return @github_event_json = nil if event_path.nil? || event_path.empty?
+
+              @github_event_json = JSON.parse(File.read(event_path))
+            rescue => e
+              Datadog.logger.error("Failed to parse GitHub event JSON: #{e}")
+              Core::Telemetry::Logger.report(e, description: "Failed to parse GitHub event JSON")
+              @github_event_json = nil
+            end
+
             def github_server_url
               return @github_server_url if defined?(@github_server_url)
 

data/lib/datadog/ci/ext/environment/providers/gitlab.rb

@@ -100,13 +100,25 @@ module Datadog
               }.to_json
             end
 
+            def git_pull_request_base_branch
+              env["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"]
+            end
+
+            def git_pull_request_base_branch_sha
+              env["CI_MERGE_REQUEST_TARGET_BRANCH_SHA"]
+            end
+
+            def git_commit_head_sha
+              env["CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"]
+            end
+
             private
 
             def extract_name_email
               return @name_email_tuple if defined?(@name_email_tuple)
 
               name_and_email_string = env["CI_COMMIT_AUTHOR"]
-              if name_and_email_string.include?("<") && (match = /^([^<]*)<([^>]*)>$/.match(name_and_email_string))
+              if name_and_email_string&.include?("<") && (match = /^([^<]*)<([^>]*)>$/.match(name_and_email_string))
                 name = match[1]
                 name = name.strip if name
                 email = match[2]

data/lib/datadog/ci/ext/environment/providers/user_defined_tags.rb

@@ -54,6 +54,18 @@ module Datadog
             def git_commit_committer_date
               env[Git::ENV_COMMIT_COMMITTER_DATE]
             end
+
+            def git_pull_request_base_branch
+              env[Git::ENV_PULL_REQUEST_BASE_BRANCH]
+            end
+
+            def git_pull_request_base_branch_sha
+              env[Git::ENV_PULL_REQUEST_BASE_BRANCH_SHA]
+            end
+
+            def git_commit_head_sha
+              env[Git::ENV_COMMIT_HEAD_SHA]
+            end
           end
         end
       end

data/lib/datadog/ci/ext/git.rb

@@ -36,6 +36,9 @@ module Datadog
         ENV_COMMIT_COMMITTER_NAME = "DD_GIT_COMMIT_COMMITTER_NAME"
         ENV_COMMIT_COMMITTER_EMAIL = "DD_GIT_COMMIT_COMMITTER_EMAIL"
         ENV_COMMIT_COMMITTER_DATE = "DD_GIT_COMMIT_COMMITTER_DATE"
+        ENV_PULL_REQUEST_BASE_BRANCH = "DD_GIT_PULL_REQUEST_BASE_BRANCH"
+        ENV_PULL_REQUEST_BASE_BRANCH_SHA = "DD_GIT_PULL_REQUEST_BASE_BRANCH_SHA"
+        ENV_COMMIT_HEAD_SHA = "DD_GIT_COMMIT_HEAD_SHA"
       end
     end
   end

data/lib/datadog/ci/ext/settings.rb

@@ -23,6 +23,9 @@ module Datadog
         ENV_TEST_MANAGEMENT_ENABLED = "DD_TEST_MANAGEMENT_ENABLED"
         ENV_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES = "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES"
         ENV_TEST_VISIBILITY_DRB_SERVER_URI = "DD_TEST_VISIBILITY_DRB_SERVER_URI"
+        ENV_AGENTLESS_LOGS_SUBMISSION_ENABLED = "DD_AGENTLESS_LOG_SUBMISSION_ENABLED"
+        ENV_AGENTLESS_LOGS_SUBMISSION_URL = "DD_AGENTLESS_LOG_SUBMISSION_URL"
+        ENV_IMPACTED_TESTS_DETECTION_ENABLED = "DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

data/lib/datadog/ci/ext/telemetry.rb

@@ -68,6 +68,8 @@ module Datadog
 
         METRIC_TEST_SESSION = "test_session"
 
+        METRIC_IMPACTED_TESTS_IS_MODIFIED = "impacted_tests.is_modified"
+
         TAG_TEST_FRAMEWORK = "test_framework"
         TAG_EVENT_TYPE = "event_type"
         TAG_HAS_CODEOWNER = "has_codeowner"
@@ -138,6 +140,8 @@ module Datadog
           GET_LOCAL_COMMITS = "get_local_commits"
           GET_OBJECTS = "get_objects"
           PACK_OBJECTS = "pack_objects"
+          DIFF = "diff"
+          BASE_COMMIT_SHA = "base_commit_sha"
         end
 
         module Provider

data/lib/datadog/ci/ext/test.rb

@@ -94,7 +94,7 @@ module Datadog
             AUTO_TEST_RETRIES_VERSION = "1"
             TEST_MANAGEMENT_QUARANTINE_VERSION = "1"
             TEST_MANAGEMENT_DISABLE_VERSION = "1"
-            TEST_MANAGEMENT_ATTEMPT_TO_FIX_VERSION = "2"
+            TEST_MANAGEMENT_ATTEMPT_TO_FIX_VERSION = "4"
           end
 
           # Map of capabilities to their versions
@@ -108,6 +108,9 @@ module Datadog
           }.freeze
         end
 
+        # impacted tests detection
+        TAG_TEST_IS_MODIFIED = "test.is_modified"
+
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
         SPAN_KIND_TEST = "test"
@@ -151,9 +154,10 @@ module Datadog
 
         # possible reasons why a test was retried
         module RetryReason
-          RETRY_NEW = "efd"
-          RETRY_FAILED = "atr"
+          RETRY_DETECT_FLAKY = "early_flake_detection"
+          RETRY_FAILED = "auto_test_retries"
           RETRY_FLAKY_FIXED = "attempt_to_fix"
+          RETRY_EXTERNAL = "external"
         end
 
         # possible reasons why a test was skipped

data/lib/datadog/ci/ext/transport.rb

@@ -28,6 +28,8 @@ module Datadog
         TEST_COVERAGE_INTAKE_HOST_PREFIX = "citestcov-intake"
         TEST_COVERAGE_INTAKE_PATH = "/api/v2/citestcov"
 
+        LOGS_INTAKE_HOST_PREFIX = "http-intake.logs"
+
         DD_API_HOST_PREFIX = "api"
 
         DD_API_SETTINGS_PATH = "/api/v2/libraries/tests/services/setting"
@@ -46,6 +48,7 @@ module Datadog
         DD_API_SETTINGS_RESPONSE_TEST_MANAGEMENT_KEY = "test_management"
         DD_API_SETTINGS_RESPONSE_ATTEMPT_TO_FIX_RETRIES_KEY = "attempt_to_fix_retries"
         DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
+        DD_API_SETTINGS_RESPONSE_IMPACTED_TESTS_ENABLED_KEY = "impacted_tests_enabled"
 
         DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"