datadog-ci 1.0.0.beta1 → 1.0.0.beta3

data/lib/datadog/ci/itr/coverage/writer.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "datadog/core/workers/async"
+require "datadog/core/workers/queue"
+require "datadog/core/workers/polling"
+
+require "datadog/core/buffer/cruby"
+require "datadog/core/buffer/thread_safe"
+
+require "datadog/core/environment/ext"
+
+module Datadog
+  module CI
+    module ITR
+      module Coverage
+        class Writer
+          include Core::Workers::Queue
+          include Core::Workers::Polling
+
+          attr_reader :transport
+
+          DEFAULT_BUFFER_MAX_SIZE = 10_000
+          DEFAULT_SHUTDOWN_TIMEOUT = 60
+
+          def initialize(transport:, options: {})
+            @transport = transport
+
+            # Workers::Polling settings
+            self.enabled = options.fetch(:enabled, true)
+
+            # Workers::Async::Thread settings
+            self.fork_policy = Core::Workers::Async::Thread::FORK_POLICY_RESTART
+
+            # Workers::IntervalLoop settings
+            self.loop_base_interval = options[:interval] if options.key?(:interval)
+            self.loop_back_off_ratio = options[:back_off_ratio] if options.key?(:back_off_ratio)
+            self.loop_back_off_max = options[:back_off_max] if options.key?(:back_off_max)
+
+            @buffer_size = options.fetch(:buffer_size, DEFAULT_BUFFER_MAX_SIZE)
+
+            self.buffer = buffer_klass.new(@buffer_size)
+
+            @shutdown_timeout = options.fetch(:shutdown_timeout, DEFAULT_SHUTDOWN_TIMEOUT)
+
+            @stopped = false
+          end
+
+          def write(event)
+            return if @stopped
+
+            # Start worker thread. If the process has forked, it will trigger #after_fork to
+            # reconfigure the worker accordingly.
+            perform
+
+            enqueue(event)
+          end
+
+          def perform(*events)
+            responses = transport.send_events(events)
+
+            if responses.find(&:server_error?)
+              loop_back_off!
+              Datadog.logger.warn { "Encountered server error while sending coverage events" }
+            end
+
+            nil
+          rescue => e
+            Datadog.logger.warn { "Error while sending coverage events: #{e}" }
+            loop_back_off!
+          end
+
+          def stop(force_stop = false, timeout = @shutdown_timeout)
+            @stopped = true
+
+            buffer.close if running?
+
+            super
+          end
+
+          def enqueue(event)
+            buffer.push(event)
+          end
+
+          def dequeue
+            buffer.pop
+          end
+
+          def work_pending?
+            !buffer.empty?
+          end
+
+          def async?
+            true
+          end
+
+          def after_fork
+            # In multiprocess environments, forks will share the same buffer until its written to.
+            # A.K.A. copy-on-write. We don't want forks to write events generated from another process.
+            # Instead, we reset it after the fork. (Make sure any enqueue operations happen after this.)
+            self.buffer = buffer_klass.new(@buffer_size)
+          end
+
+          def buffer_klass
+            if Core::Environment::Ext::RUBY_ENGINE == "ruby"
+              Core::Buffer::CRuby
+            else
+              Core::Buffer::ThreadSafe
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/itr/runner.rb

@@ -1,9 +1,18 @@
 # frozen_string_literal: true
 
+require "pp"
+
+require "datadog/core/utils/forking"
+
 require_relative "../ext/test"
 require_relative "../ext/transport"
 
-require_relative "../utils/git"
+require_relative "../git/local_repository"
+
+require_relative "../utils/parsing"
+
+require_relative "coverage/event"
+require_relative "skippable"
 
 module Datadog
   module CI
@@ -12,26 +21,46 @@ module Datadog
       # Integrates with backend to provide test impact analysis data and
       # skip tests that are not impacted by the changes
       class Runner
+        include Core::Utils::Forking
+
+        attr_reader :correlation_id, :skippable_tests, :skipped_tests_count
+
         def initialize(
+          dd_env:,
+          config_tags: {},
+          api: nil,
+          coverage_writer: nil,
           enabled: false
         )
           @enabled = enabled
+          @api = api
+          @dd_env = dd_env
+          @config_tags = config_tags || {}
+
           @test_skipping_enabled = false
           @code_coverage_enabled = false
 
+          @coverage_writer = coverage_writer
+
+          @correlation_id = nil
+          @skippable_tests = []
+
+          @skipped_tests_count = 0
+          @mutex = Mutex.new
+
           Datadog.logger.debug("ITR Runner initialized with enabled: #{@enabled}")
         end
 
-        def configure(remote_configuration, test_session)
+        def configure(remote_configuration, test_session:, git_tree_upload_worker:)
           Datadog.logger.debug("Configuring ITR Runner with remote configuration: #{remote_configuration}")
 
-          @enabled = convert_to_bool(
+          @enabled = Utils::Parsing.convert_to_bool(
             remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY, false)
           )
-          @test_skipping_enabled = @enabled && convert_to_bool(
+          @test_skipping_enabled = @enabled && Utils::Parsing.convert_to_bool(
             remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY, false)
           )
-          @code_coverage_enabled = @enabled && convert_to_bool(
+          @code_coverage_enabled = @enabled && Utils::Parsing.convert_to_bool(
             remote_configuration.fetch(Ext::Transport::DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY, false)
           )
 
@@ -46,6 +75,8 @@ module Datadog
           load_datadog_cov! if @code_coverage_enabled
 
           Datadog.logger.debug("Configured ITR Runner with enabled: #{@enabled}, skipping_tests: #{@test_skipping_enabled}, code_coverage: #{@code_coverage_enabled}")
+
+          fetch_skippable_tests(test_session: test_session, git_tree_upload_worker: git_tree_upload_worker)
         end
 
         def enabled?
@@ -60,22 +91,93 @@ module Datadog
           @code_coverage_enabled
         end
 
-        def start_coverage
+        def start_coverage(test)
           return if !enabled? || !code_coverage?
 
           coverage_collector&.start
         end
 
-        def stop_coverage(_test)
+        def stop_coverage(test)
           return if !enabled? || !code_coverage?
 
-          coverage_collector&.stop
+          coverage = coverage_collector&.stop
+          return if coverage.nil? || coverage.empty?
+
+          return if test.skipped?
+
+          test_source_file = test.source_file
+
+          # cucumber's gherkin files are not covered by the code coverage collector
+          ensure_test_source_covered(test_source_file, coverage) unless test_source_file.nil?
+
+          event = Coverage::Event.new(
+            test_id: test.id.to_s,
+            test_suite_id: test.test_suite_id.to_s,
+            test_session_id: test.test_session_id.to_s,
+            coverage: coverage
+          )
+
+          Datadog.logger.debug { "Writing coverage event \n #{event.pretty_inspect}" }
+
+          write(event)
+
+          event
+        end
+
+        def mark_if_skippable(test)
+          return if !enabled? || !skipping_tests?
+
+          skippable_test_id = Utils::TestRun.skippable_test_id(test.name, test.test_suite_name, test.parameters)
+          if @skippable_tests.include?(skippable_test_id)
+            if forked?
+              Datadog.logger.warn { "ITR is not supported for forking test runners yet" }
+              return
+            end
+
+            test.set_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR, "true")
+
+            Datadog.logger.debug { "Marked test as skippable: #{skippable_test_id}" }
+          else
+            Datadog.logger.debug { "Test is not skippable: #{skippable_test_id}" }
+          end
+        end
+
+        def count_skipped_test(test)
+          return if !test.skipped? || !test.skipped_by_itr?
+
+          if forked?
+            Datadog.logger.warn { "ITR is not supported for forking test runners yet" }
+            return
+          end
+
+          @mutex.synchronize do
+            @skipped_tests_count += 1
+          end
+        end
+
+        def write_test_session_tags(test_session)
+          return if !enabled?
+
+          Datadog.logger.debug { "Finished ITR session with test skipping enabled: #{@test_skipping_enabled}" }
+          Datadog.logger.debug { "#{@skipped_tests_count} tests were skipped" }
+
+          test_session.set_tag(Ext::Test::TAG_ITR_TESTS_SKIPPED, @skipped_tests_count.positive?.to_s)
+          test_session.set_tag(Ext::Test::TAG_ITR_TEST_SKIPPING_COUNT, @skipped_tests_count)
+        end
+
+        def shutdown!
+          @coverage_writer&.stop
         end
 
         private
 
+        def write(event)
+          # skip sending events if writer is not configured
+          @coverage_writer&.write(event)
+        end
+
         def coverage_collector
-          Thread.current[:dd_coverage_collector] ||= Coverage::DDCov.new(root: Utils::Git.root)
+          Thread.current[:dd_coverage_collector] ||= Coverage::DDCov.new(root: Git::LocalRepository.root)
         end
 
         def load_datadog_cov!
@@ -86,8 +188,29 @@ module Datadog
           @code_coverage_enabled = false
         end
 
-        def convert_to_bool(value)
-          value.to_s == "true"
+        def ensure_test_source_covered(test_source_file, coverage)
+          absolute_test_source_file_path = File.join(Git::LocalRepository.root, test_source_file)
+          return if coverage.key?(absolute_test_source_file_path)
+
+          coverage[absolute_test_source_file_path] = true
+        end
+
+        def fetch_skippable_tests(test_session:, git_tree_upload_worker:)
+          return unless skipping_tests?
+
+          # we can only request skippable tests if git metadata is already uploaded
+          git_tree_upload_worker.wait_until_done
+
+          skippable_response =
+            Skippable.new(api: @api, dd_env: @dd_env, config_tags: @config_tags)
+              .fetch_skippable_tests(test_session)
+
+          @correlation_id = skippable_response.correlation_id
+          @skippable_tests = skippable_response.tests
+
+          Datadog.logger.debug { "Fetched skippable tests: \n #{@skippable_tests}" }
+          Datadog.logger.debug { "Found #{@skippable_tests.count} skippable tests." }
+          Datadog.logger.debug { "ITR correlation ID: #{@correlation_id}" }
         end
       end
     end

data/lib/datadog/ci/itr/skippable.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "../ext/transport"
+require_relative "../ext/test"
+require_relative "../utils/test_run"
+
+module Datadog
+  module CI
+    module ITR
+      class Skippable
+        class Response
+          def initialize(http_response)
+            @http_response = http_response
+            @json = nil
+          end
+
+          def ok?
+            resp = @http_response
+            !resp.nil? && resp.ok?
+          end
+
+          def correlation_id
+            payload.dig("meta", "correlation_id")
+          end
+
+          def tests
+            res = Set.new
+
+            payload.fetch("data", [])
+              .each do |test_data|
+                next unless test_data["type"] == Ext::Test::ITR_TEST_SKIPPING_MODE
+
+                attrs = test_data["attributes"] || {}
+                res << Utils::TestRun.skippable_test_id(attrs["name"], attrs["suite"], attrs["parameters"])
+              end
+
+            res
+          end
+
+          private
+
+          def payload
+            cached = @json
+            return cached unless cached.nil?
+
+            resp = @http_response
+            return @json = {} if resp.nil? || !ok?
+
+            begin
+              @json = JSON.parse(resp.payload)
+            rescue JSON::ParserError => e
+              Datadog.logger.error("Failed to parse skippable tests response payload: #{e}. Payload was: #{resp.payload}")
+              @json = {}
+            end
+          end
+        end
+
+        def initialize(dd_env:, api: nil, config_tags: {})
+          @api = api
+          @dd_env = dd_env
+          @config_tags = config_tags
+        end
+
+        def fetch_skippable_tests(test_session)
+          api = @api
+          return Response.new(nil) unless api
+
+          request_payload = payload(test_session)
+          Datadog.logger.debug("Fetching skippable tests with request: #{request_payload}")
+
+          http_response = api.api_request(
+            path: Ext::Transport::DD_API_SKIPPABLE_TESTS_PATH,
+            payload: request_payload
+          )
+
+          Response.new(http_response)
+        end
+
+        private
+
+        def payload(test_session)
+          {
+            "data" => {
+              "type" => Ext::Transport::DD_API_SKIPPABLE_TESTS_TYPE,
+              "attributes" => {
+                "test_level" => Ext::Test::ITR_TEST_SKIPPING_MODE,
+                "service" => test_session.service,
+                "env" => @dd_env,
+                "repository_url" => test_session.git_repository_url,
+                "sha" => test_session.git_commit_sha,
+                "configurations" => {
+                  Ext::Test::TAG_OS_PLATFORM => test_session.os_platform,
+                  Ext::Test::TAG_OS_ARCHITECTURE => test_session.os_architecture,
+                  Ext::Test::TAG_OS_VERSION => test_session.os_version,
+                  Ext::Test::TAG_RUNTIME_NAME => test_session.runtime_name,
+                  Ext::Test::TAG_RUNTIME_VERSION => test_session.runtime_version,
+                  "custom" => @config_tags
+                }
+              }
+            }
+          }.to_json
+        end
+      end
+    end
+  end
+end

data/lib/datadog/ci/span.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "datadog/core/environment/platform"
+
 require_relative "ext/test"
 require_relative "utils/test_run"
 
@@ -100,6 +102,13 @@ module Datadog
         tracer_span.set_tag(key, value)
       end
 
+      # Removes tag by key.
+      # @param [String] key the key of the tag.
+      # @return [void]
+      def clear_tag(key)
+        tracer_span.clear_tag(key)
+      end
+
       # Sets metric value by key.
       # @param [String] key the key of the metric.
       # @param [Numeric] value the value of the metric.
@@ -151,6 +160,12 @@ module Datadog
         tracer_span.get_tag(Ext::Test::TAG_OS_PLATFORM)
       end
 
+      # Returns the OS version extracted from the environment.
+      # @return [String] OS version.
+      def os_version
+        tracer_span.get_tag(Ext::Test::TAG_OS_VERSION)
+      end
+
       # Returns the runtime name extracted from the environment.
       # @return [String] runtime name.
       def runtime_name
@@ -166,6 +181,7 @@ module Datadog
       def set_environment_runtime_tags
         tracer_span.set_tag(Ext::Test::TAG_OS_ARCHITECTURE, ::RbConfig::CONFIG["host_cpu"])
         tracer_span.set_tag(Ext::Test::TAG_OS_PLATFORM, ::RbConfig::CONFIG["host_os"])
+        tracer_span.set_tag(Ext::Test::TAG_OS_VERSION, Core::Environment::Platform.kernel_release)
         tracer_span.set_tag(Ext::Test::TAG_RUNTIME_NAME, Core::Environment::Ext::LANG_ENGINE)
         tracer_span.set_tag(Ext::Test::TAG_RUNTIME_VERSION, Core::Environment::Ext::ENGINE_VERSION)
         tracer_span.set_tag(Ext::Test::TAG_COMMAND, Utils::TestRun.command)

data/lib/datadog/ci/test.rb

@@ -3,6 +3,7 @@
 require "json"
 
 require_relative "span"
+require_relative "utils/test_run"
 
 module Datadog
   module CI
@@ -18,9 +19,9 @@ module Datadog
       # Finishes the current test.
       # @return [void]
       def finish
-        super
-
         recorder.deactivate_test
+
+        super
       end
 
       # Running test suite that this test is part of (if any).
@@ -62,6 +63,30 @@ module Datadog
         get_tag(Ext::Test::TAG_SOURCE_FILE)
       end
 
+      # Returns "true" if the test is skipped by the intelligent test runner.
+      # @return [Boolean] true if the test is skipped by the intelligent test runner, false otherwise.
+      def skipped_by_itr?
+        get_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR) == "true"
+      end
+
+      # Marks this test as unskippable by the intelligent test runner.
+      # This must be done before the test execution starts.
+      #
+      # Examples of tests that should be unskippable:
+      # - tests that read files from disk
+      # - tests that make network requests
+      # - tests that call external processes
+      # - tests that use forking or threading
+      #
+      # @return [void]
+      def itr_unskippable!
+        set_tag(Ext::Test::TAG_ITR_UNSKIPPABLE, "true")
+        if skipped_by_itr?
+          clear_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR)
+          set_tag(Ext::Test::TAG_ITR_FORCED_RUN, "true")
+        end
+      end
+
       # Sets the status of the span to "pass".
       # @return [void]
       def passed!
@@ -89,7 +114,7 @@ module Datadog
         record_test_result(Ext::Test::Status::SKIP)
       end
 
-      # Sets the parameters for this test (e.g. Cucumber example or RSpec shared specs).
+      # Sets the parameters for this test (e.g. Cucumber example or RSpec specs).
       # Parameters are needed to compute test fingerprint to distinguish between different tests having same names.
       #
       # @param [Hash] arguments the arguments that test accepts as key-value hash
@@ -98,15 +123,15 @@ module Datadog
       def set_parameters(arguments, metadata = {})
         return if arguments.nil?
 
-        set_tag(
-          Ext::Test::TAG_PARAMETERS,
-          JSON.generate(
-            {
-              arguments: arguments,
-              metadata: metadata
-            }
-          )
-        )
+        set_tag(Ext::Test::TAG_PARAMETERS, Utils::TestRun.test_parameters(arguments: arguments, metadata: metadata))
+      end
+
+      # Gets the parameters for this test (e.g. Cucumber example or RSpec specs) as a serialized JSON.
+      #
+      # @return [String] the serialized JSON of the parameters
+      # @return [nil] if this test does not have parameters
+      def parameters
+        get_tag(Ext::Test::TAG_PARAMETERS)
       end
 
       private

data/lib/datadog/ci/test_module.rb

@@ -14,9 +14,9 @@ module Datadog
       # Finishes this test module.
       # @return [void]
       def finish
-        super
-
         recorder.deactivate_test_module
+
+        super
       end
     end
   end

data/lib/datadog/ci/test_session.rb

@@ -15,9 +15,9 @@ module Datadog
       # Finishes the current test session.
       # @return [void]
       def finish
-        super
-
         recorder.deactivate_test_session
+
+        super
       end
 
       # Return the test session's name which is equal to test command used

data/lib/datadog/ci/test_suite.rb

@@ -26,9 +26,9 @@ module Datadog
           # we try to derive test suite status from execution stats if no status was set explicitly
           set_status_from_stats! if undefined?
 
-          super
-
           recorder.deactivate_test_suite(name)
+
+          super
         end
       end
 

data/lib/datadog/ci/test_visibility/null_recorder.rb

@@ -23,7 +23,7 @@ module Datadog
           skip_tracing(block)
         end
 
-        def trace(type, span_name, tags: {}, &block)
+        def trace(span_name, type: "span", tags: {}, &block)
           skip_tracing(block)
         end
 
@@ -42,6 +42,9 @@ module Datadog
         def active_test_suite(test_suite_name)
         end
 
+        def shutdown!
+        end
+
         private
 
         def skip_tracing(block = nil)