rspec-buildkite-analytics 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 69d1703e8bd5ea4f0d2d9d69daccbb6257b07074199c08c65ca50072f1994059
+  data.tar.gz: 6e52630ae12cf265a9ebc395e36c6c6638bc327a97c2a955eaa48e4583a08a8f
+SHA512:
+  metadata.gz: 28cfe62690f686ebb102a88165d6bff91c306305599b80937448d0a1d5823b576a50db9ae5329d0b3ce3d53d2237e5063dc22953c2cd7e2c05d6964975d1ae77
+  data.tar.gz: 22a611d02f90eb43d942374cb7586b68f817da8194d4d722385dddab070173266a1ff28be28184302956c1843df2d3d8cc915fd1094ff264dbd2954ac9440e24

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in rspec-buildkite-analytics.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    rspec-buildkite-analytics (0.2.0)
+      activesupport
+      rspec-core
+      rspec-expectations
+      websocket
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    concurrent-ruby (1.1.9)
+    diff-lcs (1.4.4)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    rake (13.0.3)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    websocket (1.2.9)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  rspec-buildkite-analytics!
+
+BUNDLED WITH
+   2.2.20

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Buildkite
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# RSpec Buildkite Analytics
+
+This gem collects data about your test suite's performance and reliability, and allows you to see trends and insights about your test suite over time ✨
+
+## Installation
+
+Add the gem to your Gemfile:
+
+```ruby
+group :test do
+  # ...
+  gem "rspec-buildkite-analytics"
+end
+```
+
+Configure your API key:
+```ruby
+RSpec::Buildkite::Analytics.configure do |config|
+  config.suite_key = "........"
+  # other config
+end
+```
+
+Run bundler to install the gem and update your `Gemfile.lock`:
+```
+$ bundle
+```
+
+Lastly, commit and push your changes to start analysing your tests:
+```
+$ git commit -m "Add Buildkite Test Analytics client"
+$ git push
+```
+
+## Contributing
+Bug reports and pull requests are welcome on GitHub at https://github.com/buildkite/rspec-buildkite-analytics.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "rspec/buildkite/analytics"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/buildkite.yaml

@@ -0,0 +1,8 @@
+steps:
+  - label: ":rspec: Tests"
+    command:
+      - "bundle"
+      - "bundle exec rake"
+    plugins:
+      - docker#v3.7.0:
+          image: ruby:latest

data/lib/rspec/buildkite/analytics/ci.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module RSpec::Buildkite::Analytics::CI
+  def self.env
+    if ENV["BUILDKITE"]
+      {
+        "CI" => "buildkite",
+        "key" => ENV["BUILDKITE_BUILD_ID"],
+        "url" => ENV["BUILDKITE_BUILD_URL"],
+        "branch" => ENV["BUILDKITE_BRANCH"],
+        "commit_sha" => ENV["BUILDKITE_COMMIT"],
+        "number" => ENV["BUILDKITE_BUILD_NUMBER"],
+        "job_id" => ENV["BUILDKITE_JOB_ID"],
+        "message" => ENV["BUILDKITE_MESSAGE"]
+      }
+    else
+      {
+        "CI" => nil,
+        "key" => SecureRandom.uuid
+      }
+    end
+  end
+end

data/lib/rspec/buildkite/analytics/network.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module RSpec::Buildkite::Analytics
+  class Network
+    module NetHTTPPatch
+      def request(request, *args, &block)
+        unless uri = request.uri
+          protocol = use_ssl? ? "https" : "http"
+          uri = URI.join("#{protocol}://#{address}:#{port}", request.path)
+        end
+
+        detail = { method: request.method.upcase, url: uri.to_s, lib: "net-http" }
+
+        http_tracer = RSpec::Buildkite::Analytics::Uploader.tracer
+        http_tracer&.enter("http", **detail)
+
+        super
+      ensure
+        http_tracer&.leave
+      end
+    end
+
+    module VCRPatch
+      def handle
+        if request_type == :stubbed_by_vcr && tracer = RSpec::Buildkite::Analytics::Uploader.tracer
+          tracer.current_span.detail.merge!(stubbed: "vcr")
+        end
+
+        super
+      end
+    end
+
+    module HTTPPatch
+      def perform(request, options)
+        detail = { method: request.verb.to_s.upcase, url: request.uri.to_s, lib: "http" }
+
+        http_tracer = RSpec::Buildkite::Analytics::Uploader.tracer
+        http_tracer&.enter("http", **detail)
+
+        super
+      ensure
+        http_tracer&.leave
+      end
+    end
+
+    module WebMockPatch
+      def response_for_request(request_signature)
+        response_from_webmock = super
+
+        if response_from_webmock && tracer = RSpec::Buildkite::Analytics::Uploader.tracer
+          tracer.current_span.detail.merge!(stubbed: "webmock")
+        end
+
+        response_from_webmock
+      end
+    end
+
+    def self.configure
+      if defined?(VCR)
+        require "vcr/request_handler"
+        VCR::RequestHandler.prepend(VCRPatch)
+      end
+
+      if defined?(WebMock)
+        WebMock::StubRegistry.prepend(WebMockPatch)
+      end
+
+      if defined?(Net) && defined?(Net::HTTP)
+        Net::HTTP.prepend(NetHTTPPatch)
+      end
+
+      if defined?(HTTP) && defined?(HTTP::Client)
+        HTTP::Client.prepend(HTTPPatch)
+      end
+    end
+  end
+end

data/lib/rspec/buildkite/analytics/object.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module RSpec::Buildkite::Analytics
+  class Object
+    module CustomObjectSleep
+      def sleep(duration)
+        tracer = RSpec::Buildkite::Analytics::Uploader.tracer
+        tracer&.enter("sleep")
+
+        super
+      ensure
+        tracer&.leave
+      end
+    end
+
+    def self.configure
+      ::Object.prepend(CustomObjectSleep)
+    end
+  end
+end

data/lib/rspec/buildkite/analytics/reporter.rb

@@ -0,0 +1,25 @@
+module RSpec::Buildkite::Analytics
+  class Reporter
+    RSpec::Core::Formatters.register self, :example_passed, :example_failed, :example_pending
+
+    def initialize(output)
+      @output = output
+    end
+
+    def handle_example(notification)
+      example = notification.example
+      trace = RSpec::Buildkite::Analytics.uploader.traces.find do |trace|
+        example.id == trace.example.id
+      end
+
+      if trace
+        trace.example = example
+        RSpec::Buildkite::Analytics.session&.write_result(trace)
+      end
+    end
+
+    alias_method :example_passed, :handle_example
+    alias_method :example_failed, :handle_example
+    alias_method :example_pending, :handle_example
+  end
+end

data/lib/rspec/buildkite/analytics/session.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require_relative "socket_connection"
+
+module RSpec::Buildkite::Analytics
+  class Session
+    # Picked 75 as the magic timeout number as it's longer than the TCP timeout of 60s 🤷‍♀️
+    CONFIRMATION_TIMEOUT = 75
+    MAX_RECONNECTION_ATTEMPTS = 3
+    WAIT_BETWEEN_RECONNECTIONS = 5
+
+    class RejectedSubscription < StandardError; end
+
+    def initialize(url, authorization_header, channel)
+      @queue = Queue.new
+      @channel = channel
+
+      @unconfirmed_idents = {}
+      @idents_mutex = Mutex.new
+      @empty = ConditionVariable.new
+      @closing = false
+      @reconnection_mutex = Mutex.new
+
+      @url = url
+      @authorization_header = authorization_header
+
+      connect
+    rescue TimeoutError => e
+      $stderr.puts "rspec-buildkite-analytics could not establish an initial connection with Buildkite. Please contact support."
+    end
+
+    def disconnected(connection)
+      reconnection_count = 0
+      @reconnection_mutex.synchronize do
+        # When the first thread detects a disconnection, it calls the disconnect method
+        # with the current connection. This thread grabs the reconnection mutex and does the
+        # reconnection, which then updates the value of @connection.
+        #
+        # At some point in that process, the second thread would have detected the
+        # disconnection too, and it also calls it with the current connection. However, the
+        # second thread can't run the reconnection code because of the mutex. By the
+        # time the mutex is released, the value of @connection has been refreshed, and so
+        # the second thread returns early and does not reattempt the reconnection.
+        return unless connection == @connection
+
+        begin
+          reconnection_count += 1
+          connect
+        rescue SocketConnection::HandshakeError, RejectedSubscription, TimeoutError, SocketConnection::SocketError => e
+          if reconnection_count > MAX_RECONNECTION_ATTEMPTS
+            $stderr.puts "rspec-buildkite-analytics experienced a disconnection and could not reconnect to Buildkite due to #{e.message}. Please contact support."
+            raise e
+          else
+            sleep(WAIT_BETWEEN_RECONNECTIONS)
+            retry
+          end
+        end
+      end
+      retransmit
+    end
+
+    def close()
+      @closing = true
+
+      # Because the server only sends us confirmations after every 10mb of
+      # data it uploads to S3, we'll never get confirmation of the
+      # identifiers of the last upload part unless we send an explicit finish,
+      # to which the server will respond with the last bits of data
+      send_eot
+
+      @idents_mutex.synchronize do
+        # Here, we sleep for 75 seconds while waiting for the server to confirm the last idents.
+        # We are woken up when the unconfirmed_idents is empty, and given back the mutex to
+        # continue operation.
+        @empty.wait(@idents_mutex, CONFIRMATION_TIMEOUT) unless @unconfirmed_idents.empty?
+      end
+
+      # Then we always disconnect cos we can't wait forever? 🤷‍♀️
+      @connection.close
+    end
+
+    def handle(_connection, data)
+      data = JSON.parse(data)
+      case data["type"]
+      when "ping"
+        # In absence of other message, the server sends us a ping every 3 seconds
+        # We are currently not doing anything with these
+      when "welcome", "confirm_subscription"
+        # Push these two messages onto the queue, so that we block on waiting for the
+        # initializing phase to complete
+        @queue.push(data)
+      when "reject_subscription"
+        raise RejectedSubscription
+      else
+        process_message(data)
+      end
+    end
+
+    def write_result(result)
+      result_as_json = result.as_json
+
+      add_unconfirmed_idents(result.id, result_as_json)
+
+      transmit_results([result_as_json])
+    end
+
+    def unconfirmed_idents_count
+      @idents_mutex.synchronize do
+        @unconfirmed_idents.count
+      end
+    end
+
+    private
+
+    def transmit_results(results_as_json)
+      @connection.transmit({
+        "identifier" => @channel,
+        "command" => "message",
+        "data" => {
+          "action" => "record_results",
+          "results" => results_as_json
+          }.to_json
+        })
+    end
+
+    def connect
+      @connection = SocketConnection.new(self, @url, {
+        "Authorization" => @authorization_header,
+      })
+
+      wait_for_welcome
+
+      @connection.transmit({
+        "command" => "subscribe",
+        "identifier" => @channel
+      })
+
+      wait_for_confirm
+    end
+
+    def pop_with_timeout
+      Timeout.timeout(30, RSpec::Buildkite::Analytics::TimeoutError, "Waited 30 seconds") do
+        @queue.pop
+      end
+    end
+
+    def wait_for_welcome
+      welcome = pop_with_timeout
+
+      if welcome && welcome != { "type" => "welcome" }
+        raise "Not a welcome: #{welcome.inspect}"
+      end
+    end
+
+    def wait_for_confirm
+      confirm = pop_with_timeout
+
+      if confirm && confirm != { "type" => "confirm_subscription", "identifier" => @channel }
+        raise "Not a confirm: #{confirm.inspect}"
+      end
+    end
+
+    def add_unconfirmed_idents(ident, data)
+      @idents_mutex.synchronize do
+        @unconfirmed_idents[ident] = data
+      end
+    end
+
+    def remove_unconfirmed_idents(idents)
+      return if idents.empty?
+
+      @idents_mutex.synchronize do
+        # Remove received idents from unconfirmed_idents
+        idents.each { |key| @unconfirmed_idents.delete(key) }
+
+        # This @empty ConditionVariable broadcasts every time that @unconfirmed_idents is
+        # empty, which will happen about every 10mb of data as that's when the server
+        # sends back confirmations.
+        #
+        # However, there aren't any threads waiting on this signal until after we
+        # send the EOT message, so the prior broadcasts shouldn't do anything.
+        @empty.broadcast if @unconfirmed_idents.empty?
+      end
+    end
+
+    def send_eot
+      # Expect server to respond with data of indentifiers last upload part
+
+      @connection.transmit({
+        "identifier" => @channel,
+        "command" => "message",
+        "data" => {
+          "action" => "end_of_transmission"
+        }.to_json
+      })
+    end
+
+    def process_message(data)
+      # Check we're getting the data we expect
+      return unless data["identifier"] == @channel
+
+      case
+      when data["message"].key?("confirm")
+        remove_unconfirmed_idents(data["message"]["confirm"])
+      else
+        # unhandled message
+      end
+    end
+
+    def retransmit
+      data = @idents_mutex.synchronize do
+        @unconfirmed_idents.values
+      end
+
+      # send the contents of the buffer, unless it's empty
+      transmit_results(data) unless data.empty?
+      # if we were disconnected in the closing phase, then resend the EOT
+      # message so the server can persist the last upload part
+      send_eot if @closing
+    end
+  end
+end

data/lib/rspec/buildkite/analytics/socket_connection.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "socket"
+require "openssl"
+require "json"
+
+module RSpec::Buildkite::Analytics
+  class SocketConnection
+    class HandshakeError < StandardError; end
+    class SocketError < StandardError; end
+
+    def initialize(session, url, headers)
+      uri = URI.parse(url)
+      @session = session
+      protocol = "http"
+
+      begin
+        socket = TCPSocket.new(uri.host, uri.port || (uri.scheme == "wss" ? 443 : 80))
+
+        if uri.scheme == "wss"
+          ctx = OpenSSL::SSL::SSLContext.new
+          protocol = "https"
+
+          ctx.min_version = :TLS1_2
+          ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER
+          ctx.cert_store = OpenSSL::X509::Store.new.tap(&:set_default_paths)
+
+          socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
+          socket.connect
+        end
+      rescue
+        # We are rescuing all here, as there are a range of Errno errors that could be
+        # raised when we fail to establish a TCP connection
+        raise SocketError
+      end
+
+      @socket = socket
+
+      headers = { "Origin" => "#{protocol}://#{uri.host}" }.merge(headers)
+      handshake = WebSocket::Handshake::Client.new(url: url, headers: headers)
+
+      @socket.write handshake.to_s
+
+      until handshake.finished?
+        if byte = @socket.getc
+          handshake << byte
+        end
+      end
+
+      # The errors below are raised when we establish the TCP connection, but get back
+      # an error, i.e. in dev we can still connect to puma-dev while nginx isn't
+      # running, or in prod we can hit a load balancer while app is down
+      unless handshake.valid?
+        case handshake.error
+        when Exception, String
+          raise HandshakeError.new(handshake.error)
+        when nil
+          raise HandshakeError.new("Invalid handshake")
+        else
+          raise HandshakeError.new(handshake.error.inspect)
+        end
+      end
+
+      @version = handshake.version
+
+      # Setting up a new thread that listens on the socket, and processes incoming
+      # comms from the server
+      @thread = Thread.new do
+        frame = WebSocket::Frame::Incoming::Client.new
+
+        while @socket
+          frame << @socket.readpartial(4096)
+
+          while data = frame.next
+            @session.handle(self, data.data)
+          end
+        end
+      rescue EOFError
+        if @socket
+          @session.disconnected(self)
+          disconnect
+        end
+      rescue IOError
+        # This is fine to ignore
+      end
+    end
+
+    def transmit(data, type: :text)
+      # this line prevents us from calling disconnect twice
+      return if @socket.nil?
+
+      raw_data = data.to_json
+      frame = WebSocket::Frame::Outgoing::Client.new(data: raw_data, type: :text, version: @version)
+      @socket.write(frame.to_s)
+    rescue Errno::EPIPE, OpenSSL::SSL::SSLError => e
+      return unless @socket
+      @session.disconnected(self)
+      disconnect
+    end
+
+    def close
+      transmit(nil, type: :close)
+      disconnect
+    end
+
+    private
+
+    def disconnect
+      socket = @socket
+      @socket = nil
+      socket&.close
+      @thread&.join unless @thread == Thread.current
+    end
+  end
+end

data/lib/rspec/buildkite/analytics/tracer.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module RSpec::Buildkite::Analytics
+  class Tracer
+    class Span
+      attr_accessor :section, :start_at, :end_at, :detail, :children
+
+      def initialize(section, start_at, end_at, detail)
+        @section = section
+        @start_at = start_at
+        @end_at = end_at
+        @detail = detail
+        @children = []
+      end
+
+      def as_json
+        {
+          section: section,
+          start_at: start_at,
+          end_at: end_at,
+          duration: end_at - start_at,
+          detail: detail,
+          children: children.map(&:as_json),
+        }
+      end
+    end
+
+    def initialize
+      @top = Span.new(:top, Concurrent.monotonic_time, nil, {})
+      @stack = [@top]
+    end
+
+    def enter(section, **detail)
+      new_entry = Span.new(section, Concurrent.monotonic_time, nil, detail)
+      current_span.children << new_entry
+      @stack << new_entry
+    end
+
+    def leave
+      current_span.end_at = Concurrent.monotonic_time
+      @stack.pop
+    end
+
+    def backfill(section, duration, **detail)
+      new_entry = Span.new(section, Concurrent.monotonic_time - duration, Concurrent.monotonic_time, detail)
+      current_span.children << new_entry
+    end
+
+    def current_span
+      @stack.last
+    end
+
+    def finalize
+      raise "Stack not empty" unless @stack.size == 1
+      @top.end_at = Concurrent.monotonic_time
+      self
+    end
+
+    def history
+      @top.as_json
+    end
+  end
+end

data/lib/rspec/buildkite/analytics/uploader.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require "rspec/expectations"
+
+require "net/http"
+require "openssl"
+require "websocket"
+
+require_relative "tracer"
+require_relative "network"
+require_relative "object"
+require_relative "session"
+require_relative "reporter"
+require_relative "ci"
+
+require "active_support"
+require "active_support/notifications"
+
+require "securerandom"
+
+module RSpec::Buildkite::Analytics
+  class Uploader
+    class Trace
+      attr_accessor :example
+      attr_reader :id, :history
+
+      def initialize(example, history)
+        @id = SecureRandom.uuid
+        @example = example
+        @history = history
+      end
+
+      def failure_message
+        case example.exception
+        when RSpec::Expectations::ExpectationNotMetError
+          example.exception.message
+        when Exception
+          "#{example.exception.class}: #{example.exception.message}"
+        end
+      end
+
+      def result_state
+        case example.execution_result.status
+        when :passed; "passed"
+        when :failed; "failed"
+        when :pending; "skipped"
+        end
+      end
+
+      def as_json
+        {
+          id: @id,
+          scope: example.example_group.metadata[:full_description],
+          name: example.description,
+          identifier: example.id,
+          location: example.location,
+          file_name: generate_file_name(example),
+          result: result_state,
+          failure: failure_message,
+          history: history,
+        }
+      end
+
+      private
+
+      def generate_file_name(example)
+        file_path_regex = /^(.*?\.rb)/
+        identifier_file_name = example.id[file_path_regex]
+        location_file_name = example.location[file_path_regex]
+
+        if identifier_file_name != location_file_name
+          # If the identifier and location files are not the same, we assume
+          # that the test was run as part of a shared example. If this isn't the
+          # case, then there's something we haven't accounted for
+          if example.metadata[:shared_group_inclusion_backtrace].any?
+            # Taking the last frame in this backtrace will give us the original
+            # entry point for the shared example
+            example.metadata[:shared_group_inclusion_backtrace].last.inclusion_location[file_path_regex]
+          else
+            "Unknown"
+          end
+        else
+          identifier_file_name
+        end
+      end
+    end
+
+    def self.traces
+      @traces ||= []
+    end
+
+    REQUEST_EXCEPTIONS = [
+      URI::InvalidURIError,
+      Net::HTTPBadResponse,
+      Net::HTTPHeaderSyntaxError,
+      Net::ReadTimeout,
+      Net::OpenTimeout,
+      OpenSSL::SSL::SSLError,
+      OpenSSL::SSL::SSLErrorWaitReadable,
+      EOFError
+    ]
+
+    def self.configure
+      RSpec::Buildkite::Analytics.uploader = self
+
+      RSpec.configure do |config|
+        config.before(:suite) do
+          config.add_formatter RSpec::Buildkite::Analytics::Reporter
+
+          if RSpec::Buildkite::Analytics.api_token
+            contact_uri = URI.parse(RSpec::Buildkite::Analytics.url)
+
+            http = Net::HTTP.new(contact_uri.host, contact_uri.port)
+            http.use_ssl = contact_uri.scheme == "https"
+
+            authorization_header = "Token token=\"#{RSpec::Buildkite::Analytics.api_token}\""
+
+            contact = Net::HTTP::Post.new(contact_uri.path, {
+              "Authorization" => authorization_header,
+              "Content-Type" => "application/json",
+            })
+            contact.body = {
+              run_env: CI.env
+            }.to_json
+
+            response = begin
+              http.request(contact)
+            rescue *REQUEST_EXCEPTIONS => e
+              puts "Error communicating with the server: #{e.message}"
+            end
+
+            if response.is_a?(Net::HTTPSuccess)
+              json = JSON.parse(response.body)
+
+              if (socket_url = json["cable"]) && (channel = json["channel"])
+                RSpec::Buildkite::Analytics.session = Session.new(socket_url, authorization_header, channel)
+              end
+            end
+          end
+        end
+
+        config.around(:each) do |example|
+          tracer = RSpec::Buildkite::Analytics::Tracer.new
+
+          # The _buildkite prefix here is added as a safeguard against name collisions
+          # as we are in the main thread
+          Thread.current[:_buildkite_tracer] = tracer
+          example.run
+          Thread.current[:_buildkite_tracer] = nil
+
+          tracer.finalize
+
+          trace = RSpec::Buildkite::Analytics::Uploader::Trace.new(example, tracer.history)
+          RSpec::Buildkite::Analytics.uploader.traces << trace
+        end
+
+        config.after(:suite) do
+          # This needs the lonely operater as the session will be nil
+          # if auth against the API token fails
+          RSpec::Buildkite::Analytics.session&.close
+        end
+      end
+
+      RSpec::Buildkite::Analytics::Network.configure
+      RSpec::Buildkite::Analytics::Object.configure
+
+      ActiveSupport::Notifications.subscribe("sql.active_record") do |name, start, finish, id, payload|
+        tracer&.backfill(:sql, finish - start, **{ query: payload[:sql] })
+      end
+    end
+
+    def self.tracer
+      Thread.current[:_buildkite_tracer]
+    end
+  end
+end

data/lib/rspec/buildkite/analytics/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Buildkite
+    module Analytics
+      VERSION = "0.2.0"
+    end
+  end
+end

data/lib/rspec/buildkite/analytics.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+require_relative "analytics/version"
+
+module RSpec::Buildkite::Analytics
+  class Error < StandardError; end
+  class TimeoutError < ::Timeout::Error; end
+
+  DEFAULT_URL = "https://analytics-api.buildkite.com/v1/uploads"
+
+  class << self
+    attr_accessor :api_token
+    attr_accessor :filename
+    attr_accessor :url
+    attr_accessor :uploader
+    attr_accessor :session
+  end
+
+  def self.configure(token: nil, url: nil, filename: nil)
+    self.api_token = token || ENV["BUILDKITE_ANALYTICS_TOKEN"]
+    self.url = url || DEFAULT_URL
+    self.filename = filename
+
+    require_relative "analytics/uploader"
+
+    self::Uploader.configure
+  end
+end

data/rspec-buildkite-analytics.gemspec

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require_relative "lib/rspec/buildkite/analytics/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "rspec-buildkite-analytics"
+  spec.version       = RSpec::Buildkite::Analytics::VERSION
+  spec.authors       = ["Buildkite"]
+  spec.email         = ["hello@buildkite.com"]
+
+  spec.summary       = "Track execution of specs and report to Buildkite Analytics"
+  spec.homepage      = "https://github.com/buildkite/rspec-buildkite-analytics"
+  spec.license       = "MIT"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/buildkite/rspec-buildkite-analytics"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.add_dependency "activesupport", '~> 6.1'
+  spec.add_dependency "rspec-core", '~> 3.10'
+  spec.add_dependency "rspec-expectations", '~> 3.10'
+  spec.add_dependency "websocket", '~> 1.2'
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: rspec-buildkite-analytics
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Buildkite
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-08-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.1'
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: rspec-expectations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: websocket
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description:
+email:
+- hello@buildkite.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- buildkite.yaml
+- lib/rspec/buildkite/analytics.rb
+- lib/rspec/buildkite/analytics/ci.rb
+- lib/rspec/buildkite/analytics/network.rb
+- lib/rspec/buildkite/analytics/object.rb
+- lib/rspec/buildkite/analytics/reporter.rb
+- lib/rspec/buildkite/analytics/session.rb
+- lib/rspec/buildkite/analytics/socket_connection.rb
+- lib/rspec/buildkite/analytics/tracer.rb
+- lib/rspec/buildkite/analytics/uploader.rb
+- lib/rspec/buildkite/analytics/version.rb
+- rspec-buildkite-analytics.gemspec
+homepage: https://github.com/buildkite/rspec-buildkite-analytics
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/buildkite/rspec-buildkite-analytics
+  source_code_uri: https://github.com/buildkite/rspec-buildkite-analytics
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: Track execution of specs and report to Buildkite Analytics
+test_files: []