buildkite-test_collector 1.0.0

data/lib/buildkite/test_collector/minitest_plugin/trace.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/indifferent_access"
+
+module Buildkite::TestCollector::MinitestPlugin
+  class Trace
+    attr_accessor :example, :failure_reason, :failure_expanded
+    attr_reader :id, :history
+
+    RESULT_CODES = {
+      '.' => 'passed',
+      'F' => 'failed',
+      'E' => 'failed',
+      'S' => 'pending',
+    }
+
+    FILE_PATH_REGEX = /^(.*?\.(rb|feature))/
+
+    def initialize(example, history:, failure_reason: nil, failure_expanded: [])
+      @id = SecureRandom.uuid
+      @example = example
+      @history = history
+      @failure_reason = failure_reason
+      @failure_expanded = failure_expanded
+    end
+
+    def result
+      RESULT_CODES[example.result_code]
+    end
+
+    def source_location
+      @source_location ||= example.method(example.name).source_location
+    end
+
+    def as_hash
+      strip_invalid_utf8_chars(
+        id: id,
+        scope: example.class.name,
+        name: example.name,
+        identifier: identifier,
+        location: location,
+        file_name: file_name,
+        result: result,
+        failure_reason: failure_reason,
+        failure_expanded: failure_expanded,
+        history: history,
+      ).with_indifferent_access.compact
+    end
+
+    private
+
+    def location
+      if file_name
+        "#{file_name}:#{line_number}"
+      end
+    end
+    alias_method :identifier, :location
+
+    def file_name
+      @file_name ||= File.join('./', source_location[0].delete_prefix(project_dir))
+    end
+
+    def line_number
+      @line_number ||= source_location[1]
+    end
+
+    def project_dir
+      if defined?(Rails) && Rails.respond_to?(:root)
+        Rails.root
+      else
+        Dir.getwd
+      end
+    end
+
+    def failure_reason
+      @failure_reason ||= example.failure&.message
+    end
+
+    def failure_expanded
+      @failure_expanded ||= begin
+        example.failures.map do |failure|
+          {
+            expanded: failure.message,
+            backtrace: failure.backtrace,
+          }
+        end
+      end
+    end
+
+    def strip_invalid_utf8_chars(object)
+      if object.is_a?(Hash)
+        Hash[object.map { |key, value| [key, strip_invalid_utf8_chars(value)] }]
+      elsif object.is_a?(Array)
+        object.map { |value| strip_invalid_utf8_chars(value) }
+      elsif object.is_a?(String)
+        object.encode('UTF-8', :invalid => :replace, :undef => :replace)
+      else
+        object
+      end
+    end
+  end
+end

data/lib/buildkite/test_collector/minitest_plugin.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative "minitest_plugin/reporter"
+require_relative "minitest_plugin/trace"
+
+module Buildkite::TestCollector::MinitestPlugin
+  def before_setup
+    super
+    tracer = Buildkite::TestCollector::Tracer.new
+    # The _buildkite prefix here is added as a safeguard against name collisions
+    # as we are in the main thread
+    Thread.current[:_buildkite_tracer] = tracer
+  end
+
+  def before_teardown
+    super
+
+    tracer = Thread.current[:_buildkite_tracer]
+    if !tracer.nil?
+      Thread.current[:_buildkite_tracer] = nil
+      tracer.finalize
+
+      trace = Buildkite::TestCollector::MinitestPlugin::Trace.new(self, history: tracer.history)
+      Buildkite::TestCollector.uploader.traces[trace.source_location] = trace
+    end
+  end
+end

data/lib/buildkite/test_collector/network.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Buildkite::TestCollector
+  class Network
+    module NetHTTPPatch
+      def request(request, *args, &block)
+        unless uri = request.uri
+          protocol = use_ssl? ? "https" : "http"
+          uri = URI.join("#{protocol}://#{address}:#{port}", request.path)
+        end
+
+        detail = { method: request.method.upcase, url: uri.to_s, lib: "net-http" }
+
+        http_tracer = Buildkite::TestCollector::Uploader.tracer
+        http_tracer&.enter("http", **detail)
+
+        super
+      ensure
+        http_tracer&.leave
+      end
+    end
+
+    module VCRPatch
+      def handle
+        if request_type == :stubbed_by_vcr && tracer = Buildkite::TestCollector::Uploader.tracer
+          tracer.current_span.detail.merge!(stubbed: "vcr")
+        end
+
+        super
+      end
+    end
+
+    module HTTPPatch
+      def perform(request, options)
+        detail = { method: request.verb.to_s.upcase, url: request.uri.to_s, lib: "http" }
+
+        http_tracer = Buildkite::TestCollector::Uploader.tracer
+        http_tracer&.enter("http", **detail)
+
+        super
+      ensure
+        http_tracer&.leave
+      end
+    end
+
+    module WebMockPatch
+      def response_for_request(request_signature)
+        response_from_webmock = super
+
+        if response_from_webmock && tracer = Buildkite::TestCollector::Uploader.tracer
+          tracer.current_span.detail.merge!(stubbed: "webmock")
+        end
+
+        response_from_webmock
+      end
+    end
+
+    def self.configure
+      if defined?(VCR)
+        require "vcr/request_handler"
+        VCR::RequestHandler.prepend(VCRPatch)
+      end
+
+      if defined?(WebMock)
+        WebMock::StubRegistry.prepend(WebMockPatch)
+      end
+
+      if defined?(Net) && defined?(Net::HTTP)
+        Net::HTTP.prepend(NetHTTPPatch)
+      end
+
+      if defined?(HTTP) && defined?(HTTP::Client)
+        HTTP::Client.prepend(HTTPPatch)
+      end
+    end
+  end
+end

data/lib/buildkite/test_collector/object.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Buildkite::TestCollector
+  class Object
+    module CustomObjectSleep
+      def sleep(duration)
+        tracer = Buildkite::TestCollector::Uploader.tracer
+        tracer&.enter("sleep")
+
+        super
+      ensure
+        tracer&.leave
+      end
+    end
+
+    def self.configure
+      ::Object.prepend(CustomObjectSleep)
+    end
+  end
+end

data/lib/buildkite/test_collector/rspec_plugin/reporter.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Buildkite::TestCollector::RSpecPlugin
+  class Reporter
+    RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending, :dump_summary
+
+    attr_reader :output
+
+    def initialize(output)
+      @output = output
+    end
+
+    def handle_example(notification)
+      example = notification.example
+      trace = Buildkite::TestCollector.uploader.traces[example.id]
+
+      if trace
+        trace.example = example
+        if example.execution_result.status == :failed
+          trace.failure_reason, trace.failure_expanded = failure_info(notification)
+        end
+        Buildkite::TestCollector.session&.write_result(trace)
+      end
+    end
+
+    def dump_summary(notification)
+      if Buildkite::TestCollector.session.present?
+        examples_count = {
+          examples: notification.examples.count,
+          failed: notification.failed_examples.count,
+          pending: notification.pending_examples.count,
+          errors_outside_examples: notification.errors_outside_of_examples_count
+        }
+
+        Buildkite::TestCollector.session.close(examples_count)
+      end
+    end
+
+    alias_method :example_passed, :handle_example
+    alias_method :example_failed, :handle_example
+    alias_method :example_pending, :handle_example
+
+    private
+
+    MULTIPLE_ERRORS = [
+      RSpec::Expectations::MultipleExpectationsNotMetError,
+      RSpec::Core::MultipleExceptionError
+    ]
+
+    def failure_info(notification)
+      failure_expanded = []
+
+      if Buildkite::TestCollector::RSpecPlugin::Reporter::MULTIPLE_ERRORS.include?(notification.exception.class)
+        failure_reason = notification.exception.summary
+        notification.exception.all_exceptions.each do |exception|
+          # an example with multiple failures doesn't give us a
+          # separate message lines and backtrace object to send, so
+          # I've reached into RSpec internals and duplicated the
+          # construction of these
+          message_lines = RSpec::Core::Formatters::ExceptionPresenter.new(exception, notification.example).colorized_message_lines
+
+          failure_expanded << {
+            expanded: format_message_lines(message_lines),
+            backtrace:  RSpec.configuration.backtrace_formatter.format_backtrace(exception.backtrace)
+          }
+        end
+      else
+        message_lines = notification.colorized_message_lines
+        failure_reason = strip_diff_colors(message_lines.shift)
+
+        failure_expanded << {
+          expanded:  format_message_lines(message_lines),
+          backtrace: notification.formatted_backtrace
+        }
+      end
+
+      return failure_reason, failure_expanded
+    end
+
+    def format_message_lines(message_lines)
+      message_lines.map! { |l| strip_diff_colors(l) }
+      # the first line is sometimes blank, depending on the error reported
+      message_lines.shift if message_lines.first.blank?
+      # the last line is sometimes blank, depending on the error reported
+      message_lines.pop if message_lines.last.blank?
+      message_lines
+    end
+
+    def strip_diff_colors(string)
+      string.gsub(/\e\[([;\d]+)?m/, '')
+    end
+  end
+end

data/lib/buildkite/test_collector/rspec_plugin/trace.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/indifferent_access"
+
+module Buildkite::TestCollector::RSpecPlugin
+  class Trace
+    attr_accessor :example, :failure_reason, :failure_expanded
+    attr_reader :id, :history
+
+    FILE_PATH_REGEX = /^(.*?\.(rb|feature))/
+
+    def initialize(example, history:, failure_reason: nil, failure_expanded: [])
+      @id = SecureRandom.uuid
+      @example = example
+      @history = history
+      @failure_reason = failure_reason
+      @failure_expanded = failure_expanded
+    end
+
+    def result
+      case example.execution_result.status
+      when :passed; "passed"
+      when :failed; "failed"
+      when :pending; "skipped"
+      end
+    end
+
+    def as_hash
+      strip_invalid_utf8_chars(
+        id: id,
+        scope: example.example_group.metadata[:full_description],
+        name: example.description,
+        identifier: example.id,
+        location: example.location,
+        file_name: file_name,
+        result: result,
+        failure_reason: failure_reason,
+        failure_expanded: failure_expanded,
+        history: history,
+      ).with_indifferent_access.compact
+    end
+
+    private
+
+    def file_name
+      @file_name ||= begin
+        identifier_file_name = strip_invalid_utf8_chars(example.id)[FILE_PATH_REGEX]
+        location_file_name = example.location[FILE_PATH_REGEX]
+
+        if identifier_file_name != location_file_name
+          # If the identifier and location files are not the same, we assume
+          # that the test was run as part of a shared example. If this isn't the
+          # case, then there's something we haven't accounted for
+          if shared_example?
+            # Taking the last frame in this backtrace will give us the original
+            # entry point for the shared example
+            shared_example_call_location[FILE_PATH_REGEX]
+          else
+            "Unknown"
+          end
+        else
+          identifier_file_name
+        end
+      end
+    end
+
+    def shared_example?
+      example.metadata[:shared_group_inclusion_backtrace].any?
+    end
+
+    def shared_example_call_location
+      example.metadata[:shared_group_inclusion_backtrace].last.inclusion_location
+    end
+
+    def strip_invalid_utf8_chars(object)
+      if object.is_a?(Hash)
+        Hash[object.map { |key, value| [key, strip_invalid_utf8_chars(value)] }]
+      elsif object.is_a?(Array)
+        object.map { |value| strip_invalid_utf8_chars(value) }
+      elsif object.is_a?(String)
+        object.encode('UTF-8', :invalid => :replace, :undef => :replace)
+      else
+        object
+      end
+    end
+  end
+end