rspec-buildkite-analytics 0.4.0 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44dd3ce4c2f47452f1eb52c96a2b86a7f49dd971d38350152b773dd343a85969
-  data.tar.gz: 81fad9923f72c5a4e733b9e93917323c1037a6a0fa9ba4c0f76e9c3d0820f06b
+  metadata.gz: 499229f993ea2f8556a6d908f011325bf989de60c9a69a266338356b9f89426f
+  data.tar.gz: 901fec67b35b6ca85f61900a4cd9fa00293959892eb45e95339b6d4a6c27f527
 SHA512:
-  metadata.gz: 6eb38e960fa8c7c8e6b4ff29070d188db20f0868c316a7b2b51051cd733793c8c86ea736385ddefa3eab04afb585d20ebef5fcf3bb7d3a2ee62513d9dcf448d5
-  data.tar.gz: dec76a5f780c47f7f340b83501450545ccd810dbf9b0c43755d4fff300197da750b9411a49eb757f41a094d7058780a78f7a6d859fef34514105be6e97fd8765
+  metadata.gz: 0b7942d5c6b1abd45ff662cd52878d348bb1f9b3e2fa9fd1a6285eb7f521ac1d1f9dcc52007650dde979da9ddcbdd0506afe869708c785d033f9cf2e969eb009
+  data.tar.gz: c5484d6a416ac0faf663390c64d86e6c24184ddadbaa286b66407afd169444b7af9a6b9e665ead3438e87e719b5563ea0bc184c963643e17b13cdb8d3517ad1c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rspec-buildkite-analytics (0.4.0)
+    rspec-buildkite-analytics (0.6.3)
       activesupport (>= 5.2, <= 7.0)
       rspec-core (~> 3.10)
       rspec-expectations (~> 3.10)
@@ -10,18 +10,17 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (7.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     concurrent-ruby (1.1.9)
     diff-lcs (1.4.4)
-    i18n (1.8.10)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    rake (13.0.3)
+    minitest (5.15.0)
+    rake (13.0.6)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
@@ -31,14 +30,13 @@ GEM
     rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.1)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.1)
+    rspec-support (3.10.3)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     websocket (1.2.9)
-    zeitwerk (2.5.1)
 
 PLATFORMS
   ruby

data/lib/rspec/buildkite/analytics/ci.rb

@@ -14,13 +14,17 @@ module RSpec::Buildkite::Analytics::CI
         "number" => ENV["BUILDKITE_BUILD_NUMBER"],
         "job_id" => ENV["BUILDKITE_JOB_ID"],
         "message" => ENV["BUILDKITE_MESSAGE"],
-        "debug" => ENV["BUILDKITE_ANALYTICS_DEBUG_ENABLED"]
+        "debug" => ENV["BUILDKITE_ANALYTICS_DEBUG_ENABLED"],
+        "version" => RSpec::Buildkite::Analytics::VERSION,
+        "collector" => RSpec::Buildkite::Analytics::NAME
       }
     else
       {
         "CI" => nil,
         "key" => SecureRandom.uuid,
-        "debug" => ENV["BUILDKITE_ANALYTICS_DEBUG_ENABLED"]
+        "debug" => ENV["BUILDKITE_ANALYTICS_DEBUG_ENABLED"],
+        "version" => RSpec::Buildkite::Analytics::VERSION,
+        "collector" => RSpec::Buildkite::Analytics::NAME
       }
     end
   end

data/lib/rspec/buildkite/analytics/reporter.rb

@@ -1,3 +1,5 @@
+require "time"
+
 module RSpec::Buildkite::Analytics
   class Reporter
     RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending, :dump_summary
@@ -16,6 +18,7 @@ module RSpec::Buildkite::Analytics
 
       if trace
         trace.example = example
+        trace.failure_reason, trace.failure_expanded = failure_info(notification) if example.execution_result.status == :failed
         RSpec::Buildkite::Analytics.session&.write_result(trace)
       end
     end
@@ -33,7 +36,7 @@ module RSpec::Buildkite::Analytics
 
         # Write the debug file, if debug mode is enabled
         if RSpec::Buildkite::Analytics.debug_enabled
-          filename = "#{RSpec::Buildkite::Analytics.debug_filepath}/bk-analytics-#{DateTime.current.strftime("%F-%R:%S")}-#{ENV["BUILDKITE_JOB_ID"]}.log.gz"
+          filename = "#{RSpec::Buildkite::Analytics.debug_filepath}/bk-analytics-#{Time.now.strftime("%F-%R:%S")}-#{ENV["BUILDKITE_JOB_ID"]}.log.gz"
 
           File.open(filename, "wb") do |f|
             gz = Zlib::GzipWriter.new(f)
@@ -43,9 +46,59 @@ module RSpec::Buildkite::Analytics
         end
       end
     end
- 
+
     alias_method :example_passed, :handle_example
     alias_method :example_failed, :handle_example
     alias_method :example_pending, :handle_example
+
+    private
+
+    MULTIPLE_ERRORS = [
+      RSpec::Expectations::MultipleExpectationsNotMetError,
+      RSpec::Core::MultipleExceptionError
+    ]
+
+    def failure_info(notification)
+      failure_expanded = []
+
+      if RSpec::Buildkite::Analytics::Reporter::MULTIPLE_ERRORS.include?(notification.exception.class)
+        failure_reason = notification.exception.summary
+        notification.exception.all_exceptions.each do |exception|
+          # an example with multiple failures doesn't give us a
+          # separate message lines and backtrace object to send, so
+          # I've reached into RSpec internals and duplicated the
+          # construction of these
+          message_lines = RSpec::Core::Formatters::ExceptionPresenter.new(exception, notification.example).colorized_message_lines
+
+          failure_expanded << {
+            expanded: format_message_lines(message_lines),
+            backtrace:  RSpec.configuration.backtrace_formatter.format_backtrace(exception.backtrace)
+          }
+        end
+      else
+        message_lines = notification.colorized_message_lines
+        failure_reason = strip_diff_colors(message_lines.shift)
+
+        failure_expanded << {
+          expanded:  format_message_lines(message_lines),
+          backtrace: notification.formatted_backtrace
+        }
+      end
+
+      return failure_reason, failure_expanded
+    end
+
+    def format_message_lines(message_lines)
+      message_lines.map! { |l| strip_diff_colors(l) }
+      # the first line is sometimes blank, depending on the error reported
+      message_lines.shift if message_lines.first.blank?
+      # the last line is sometimes blank, depending on the error reported
+      message_lines.pop if message_lines.last.blank?
+      message_lines
+    end
+
+    def strip_diff_colors(string)
+      string.gsub(/\e\[([;\d]+)?m/, '')
+    end
   end
 end

data/lib/rspec/buildkite/analytics/session.rb

@@ -35,13 +35,16 @@ module RSpec::Buildkite::Analytics
     attr_reader :logger
 
     def initialize(url, authorization_header, channel)
-      @queue = Queue.new
+      @establish_subscription_queue = Queue.new
       @channel = channel
 
       @unconfirmed_idents = {}
       @idents_mutex = Mutex.new
+      @send_queue = Queue.new
       @empty = ConditionVariable.new
       @closing = false
+      @eot_queued = false
+      @eot_queued_mutex = Mutex.new
       @reconnection_mutex = Mutex.new
 
       @url = url
@@ -49,13 +52,25 @@ module RSpec::Buildkite::Analytics
 
       @logger = Logger.new
 
-      connect
-    rescue TimeoutError, InitialConnectionFailure => e
-      $stderr.puts "rspec-buildkite-analytics could not establish an initial connection with Buildkite due to #{e.message}. You may be missing some data for this test suite, please contact support."
+      reconnection_count = 0
+
+      begin
+        reconnection_count += 1
+        connect
+      rescue TimeoutError, InitialConnectionFailure => e
+        @logger.write("rspec-buildkite-analytics could not establish an initial connection with Buildkite due to #{e}. Attempting retry #{reconnection_count} of #{MAX_RECONNECTION_ATTEMPTS}...")
+        if reconnection_count > MAX_RECONNECTION_ATTEMPTS
+          $stderr.puts "rspec-buildkite-analytics could not establish an initial connection with Buildkite due to #{e.message} after #{MAX_RECONNECTION_ATTEMPTS} attempts. You may be missing some data for this test suite, please contact support if this issue persists."
+        else
+          sleep(WAIT_BETWEEN_RECONNECTIONS)
+          @logger.write("retrying reconnection")
+          retry
+        end
+      end
+      init_write_thread
     end
 
     def disconnected(connection)
-      reconnection_count = 0
       @reconnection_mutex.synchronize do
         # When the first thread detects a disconnection, it calls the disconnect method
         # with the current connection. This thread grabs the reconnection mutex and does the
@@ -69,10 +84,13 @@ module RSpec::Buildkite::Analytics
         return unless connection == @connection
         @logger.write("starting reconnection")
 
+        reconnection_count = 0
+
         begin
           reconnection_count += 1
           connect
-        rescue SocketConnection::HandshakeError, RejectedSubscription, TimeoutError, SocketConnection::SocketError => e
+          init_write_thread
+        rescue SocketConnection::HandshakeError, RejectedSubscription, TimeoutError, InitialConnectionFailure, SocketConnection::SocketError => e
           @logger.write("failed reconnection attempt #{reconnection_count} due to #{e}")
           if reconnection_count > MAX_RECONNECTION_ATTEMPTS
             $stderr.puts "rspec-buildkite-analytics experienced a disconnection and could not reconnect to Buildkite due to #{e.message}. Please contact support."
@@ -100,7 +118,9 @@ module RSpec::Buildkite::Analytics
 
       @idents_mutex.synchronize do
         @logger.write("waiting for last confirm")
-        # Here, we sleep for 75 seconds while waiting for the server to confirm the last idents.
+        # Here, we sleep for 75 seconds while waiting for the send
+        # queue to be drained and for the server to confirm the last
+        # idents.
         # We are woken up when the unconfirmed_idents is empty, and given back the mutex to
         # continue operation.
         @empty.wait(@idents_mutex, CONFIRMATION_TIMEOUT) unless @unconfirmed_idents.empty?
@@ -108,6 +128,8 @@ module RSpec::Buildkite::Analytics
 
       # Then we always disconnect cos we can't wait forever? 🤷‍♀️
       @connection.close
+      # We kill the write thread cos it's got a while loop in it, so it won't finish otherwise
+      @write_thread&.kill
       @logger.write("socket connection closed")
     end
 
@@ -121,7 +143,7 @@ module RSpec::Buildkite::Analytics
       when "welcome", "confirm_subscription"
         # Push these two messages onto the queue, so that we block on waiting for the
         # initializing phase to complete
-        @queue.push(data)
+        @establish_subscription_queue.push(data)
       @logger.write("received #{data['type']}")
       when "reject_subscription"
         @logger.write("received rejected_subscription")
@@ -132,13 +154,9 @@ module RSpec::Buildkite::Analytics
     end
 
     def write_result(result)
-      result_as_json = result.as_json
-
-      add_unconfirmed_idents(result.id, result_as_json)
-
-      transmit_results([result_as_json])
+      queue_and_track_result(result.id, result.as_hash)
 
-      @logger.write("transmitted #{result.id}")
+      @logger.write("added #{result.id} to send queue")
     end
 
     def unconfirmed_idents_count
@@ -149,17 +167,6 @@ module RSpec::Buildkite::Analytics
 
     private
 
-    def transmit_results(results_as_json)
-      @connection.transmit({
-        "identifier" => @channel,
-        "command" => "message",
-        "data" => {
-          "action" => "record_results",
-          "results" => results_as_json
-          }.to_json
-        })
-    end
-
     def connect
       @logger.write("starting socket connection process")
 
@@ -179,9 +186,52 @@ module RSpec::Buildkite::Analytics
       @logger.write("connected")
     end
 
+    def init_write_thread
+      # As this method can be called multiple times in the
+      # reconnection process, kill prev write threads (if any) before
+      # setting up the new one
+      @write_thread&.kill
+
+      @write_thread = Thread.new do
+        @logger.write("hello from write thread")
+        # Pretty sure this eternal loop is fine cos the call to queue.pop is blocking
+        loop do
+          data = @send_queue.pop
+          message_type = data["action"]
+
+          if message_type == "end_of_transmission"
+            # Because of the unpredictable sequencing between the test suite finishing
+            # (EOT gets queued) and disconnections happening (retransmit results gets
+            # queued), we don't want to send an EOT before any retransmits are sent.
+            if @send_queue.length > 0
+              @send_queue << data
+              @logger.write("putting eot at back of queue")
+              next
+            end
+            @eot_queued_mutex.synchronize do
+              @eot_queued = false
+            end
+          end
+
+          @connection.transmit({
+            "identifier" => @channel,
+            "command" => "message",
+            "data" => data.to_json
+          })
+
+          if RSpec::Buildkite::Analytics.debug_enabled
+            ids = if message_type == "record_results"
+              data["results"].map { |result| result["id"] }
+            end
+            @logger.write("transmitted #{message_type} #{ids}")
+          end
+        end
+      end
+    end
+
     def pop_with_timeout(message_type)
       Timeout.timeout(30, RSpec::Buildkite::Analytics::TimeoutError, "Timeout: Waited 30 seconds for #{message_type}") do
-        @queue.pop
+        @establish_subscription_queue.pop
       end
     end
 
@@ -201,9 +251,16 @@ module RSpec::Buildkite::Analytics
       end
     end
 
-    def add_unconfirmed_idents(ident, data)
+    def queue_and_track_result(ident, result_as_hash)
       @idents_mutex.synchronize do
-        @unconfirmed_idents[ident] = data
+        @unconfirmed_idents[ident] = result_as_hash
+
+        data = {
+          "action" => "record_results",
+          "results" => [result_as_hash]
+        }
+
+        @send_queue << data
       end
     end
 
@@ -222,23 +279,29 @@ module RSpec::Buildkite::Analytics
         #
         # However, there aren't any threads waiting on this signal until after we
         # send the EOT message, so the prior broadcasts shouldn't do anything.
-        @empty.broadcast if @unconfirmed_idents.empty?
+        if @unconfirmed_idents.empty?
+          @empty.broadcast
+          @logger.write("broadcast empty")
+        else
+          @logger.write("still waiting on confirm for #{@unconfirmed_idents.keys}")
+        end
       end
     end
 
     def send_eot
-      # Expect server to respond with data of indentifiers last upload part
-
-      @connection.transmit({
-        "identifier" => @channel,
-        "command" => "message",
-        "data" => {
+      @eot_queued_mutex.synchronize do
+        return if @eot_queued
+        # Expect server to respond with data of indentifiers last upload part
+        data = {
           "action" => "end_of_transmission",
           "examples_count" => @examples_count.to_json
-        }.to_json
-      })
+        }
 
-      @logger.write("transmitted EOT")
+        @send_queue << data
+        @eot_queued = true
+
+        @logger.write("added EOT to send queue")
+      end
     end
 
     def process_message(data)
@@ -255,14 +318,19 @@ module RSpec::Buildkite::Analytics
     end
 
     def retransmit
-      data = @idents_mutex.synchronize do
-        @unconfirmed_idents.values
-      end
+      @idents_mutex.synchronize do
+        results = @unconfirmed_idents.values
 
-      # send the contents of the buffer, unless it's empty
-      unless data.empty?
-        @logger.write("retransmitting data")
-        transmit_results(data)
+        # queue the contents of the buffer, unless it's empty
+        unless results.empty?
+          data = {
+            "action" => "record_results",
+            "results" => results
+          }
+
+          @send_queue << data
+          @logger.write("queueing up retransmitted results #{@unconfirmed_idents.keys}")
+        end
       end
 
       # if we were disconnected in the closing phase, then resend the EOT

data/lib/rspec/buildkite/analytics/socket_connection.rb

@@ -110,6 +110,7 @@ module RSpec::Buildkite::Analytics
       @socket.write(frame.to_s)
     rescue Errno::EPIPE, Errno::ECONNRESET, OpenSSL::SSL::SSLError => e
       return unless @socket
+      return if type == :close
       @session.logger.write("got #{e}, attempting disconnected flow")
       @session.disconnected(self)
       disconnect

data/lib/rspec/buildkite/analytics/tracer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "active_support/core_ext/hash/indifferent_access"
+
 module RSpec::Buildkite::Analytics
   class Tracer
     class Span
@@ -13,15 +15,15 @@ module RSpec::Buildkite::Analytics
         @children = []
       end
 
-      def as_json
+      def as_hash
         {
           section: section,
           start_at: start_at,
           end_at: end_at,
           duration: end_at - start_at,
           detail: detail,
-          children: children.map(&:as_json),
-        }
+          children: children.map(&:as_hash),
+        }.with_indifferent_access
       end
     end
 
@@ -57,7 +59,7 @@ module RSpec::Buildkite::Analytics
     end
 
     def history
-      @top.as_json
+      @top.as_hash
     end
   end
 end

data/lib/rspec/buildkite/analytics/uploader.rb

@@ -22,22 +22,15 @@ require "securerandom"
 module RSpec::Buildkite::Analytics
   class Uploader
     class Trace
-      attr_accessor :example
+      attr_accessor :example, :failure_reason, :failure_expanded
       attr_reader :id, :history
 
       def initialize(example, history)
         @id = SecureRandom.uuid
         @example = example
         @history = history
-      end
-
-      def failure_message
-        case example.exception
-        when RSpec::Expectations::ExpectationNotMetError
-          example.exception.message
-        when Exception
-          "#{example.exception.class}: #{example.exception.message}"
-        end
+        @failure_reason = nil
+        @failure_expanded = []
       end
 
       def result_state
@@ -48,7 +41,7 @@ module RSpec::Buildkite::Analytics
         end
       end
 
-      def as_json
+      def as_hash
         {
           id: @id,
           scope: example.example_group.metadata[:full_description],
@@ -57,9 +50,10 @@ module RSpec::Buildkite::Analytics
           location: example.location,
           file_name: generate_file_name(example),
           result: result_state,
-          failure: failure_message,
+          failure_reason: failure_reason,
+          failure_expanded: failure_expanded,
           history: history,
-        }
+        }.with_indifferent_access.compact
       end
 
       private
@@ -121,7 +115,8 @@ module RSpec::Buildkite::Analytics
               "Content-Type" => "application/json",
             })
             contact.body = {
-              run_env: CI.env
+              run_env: CI.env,
+              format: "websocket"
             }.to_json
 
             response = begin
@@ -130,6 +125,8 @@ module RSpec::Buildkite::Analytics
               puts "Buildkite Test Analytics: Error communicating with the server: #{e.message}"
             end
 
+            return unless response
+
             case response.code
             when "401"
               puts "Buildkite Test Analytics: Invalid Suite API key. Please double check your Suite API key."
@@ -141,10 +138,12 @@ module RSpec::Buildkite::Analytics
               end
             else
               request_id = response.to_hash["x-request-id"]
-              puts "Buildkite Test Analytics: Unknown error. If this error persists, please contact support+analytics@buildkite.com with this request ID `#{request_id}`."
+              puts "rspec-buildkite-analytics could not establish an initial connection with Buildkite. You may be missing some data for this test suite, please contact support."
             end
           else
-            puts "Buildkite Test Analytics: No Suite API key provided. You can get the API key from your Suite settings page."
+            if !!ENV["BUILDKITE_BUILD_ID"]
+              puts "Buildkite Test Analytics: No Suite API key provided. You can get the API key from your Suite settings page."
+            end
           end
         end
 

data/lib/rspec/buildkite/analytics/version.rb

@@ -3,7 +3,8 @@
 module RSpec
   module Buildkite
     module Analytics
-      VERSION = "0.4.0"
+      VERSION = "0.6.3"
+      NAME = "rspec-buildkite"
     end
   end
 end

data/lib/rspec/buildkite/analytics.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "timeout"
+require "tmpdir"
 
 require_relative "analytics/version"
 
@@ -23,7 +24,7 @@ module RSpec::Buildkite::Analytics
     self.api_token = token || ENV["BUILDKITE_ANALYTICS_TOKEN"]
     self.url = url || DEFAULT_URL
     self.debug_enabled = debug_enabled || !!(ENV["BUILDKITE_ANALYTICS_DEBUG_ENABLED"])
-    self.debug_filepath = debug_filepath || ENV["BUILDKITE_ANALYTICS_DEBUG_FILEPATH"]
+    self.debug_filepath = debug_filepath || ENV["BUILDKITE_ANALYTICS_DEBUG_FILEPATH"] || Dir.tmpdir
 
     require_relative "analytics/uploader"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec-buildkite-analytics
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.6.3
 platform: ruby
 authors:
 - Buildkite
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-11-04 00:00:00.000000000 Z
+date: 2022-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport