rspec_trunk_flaky_tests 0.12.9.pre.beta.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '00089c490aca3f0766ade829dd6e96620cc900a7173fef7851d6844a8ebb7375'
+  data.tar.gz: 9a62065dccb28f61dbb2d440d6880a65b6e2174d4ae86634158fff88ab3cd5cb
+SHA512:
+  metadata.gz: 561359db55239e235de3929888c2aa779d26afc05862477723c764e4821c9760d19dd44195e87bfa2f771fbb242392224b297944d861f5ccfd814d4a4ad22269
+  data.tar.gz: bf4fca81d43e248167cc4c3b4bc8c8495c87083f2c9c43201e9333caa0e48017f8948b2a6a01c02174f2a022b62f2f4372cdf792f8c417e877072e00b60f13ad

data/lib/rspec_trunk_flaky_tests.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+begin
+  ruby_version = /(\d+\.\d+)/.match(RUBY_VERSION)
+  require_relative "rspec_trunk_flaky_tests/#{ruby_version}/rspec_trunk_flaky_tests"
+rescue LoadError
+  begin
+    require_relative 'rspec_trunk_flaky_tests/rspec_trunk_flaky_tests'
+  rescue LoadError
+    raise LoadError, <<~MSG
+      Could not load the native extension for rspec_trunk_flaky_tests.
+
+      You are using the pure ruby variant of this gem, which is a placeholder
+      that exists so that Gemfile.lock files with `PLATFORMS ruby` can resolve
+      this dependency. It does not include a precompiled native extension.
+
+      Precompiled native gems are available for:
+        x86_64-linux, aarch64-linux, arm64-darwin, x86_64-darwin
+
+      If you are on a supported platform and seeing this error, make sure
+      bundler is selecting the native variant for your platform:
+
+          bundle lock --add-platform #{RUBY_PLATFORM}
+          bundle install
+
+      If you are on an unsupported platform, this gem cannot be used in your
+      environment. Please open an issue at:
+        https://github.com/trunk-io/analytics-cli/issues
+
+      Current platform: #{RUBY_PLATFORM}
+      Current Ruby version: #{RUBY_VERSION}
+    MSG
+  end
+end

data/lib/trunk_spec_helper.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+# Trunk RSpec Helper
+#
+# This helper integrates Trunk Flaky Tests with RSpec to automatically
+# quarantine flaky tests and upload test results.
+#
+# Required environment variables:
+#   TRUNK_ORG_URL_SLUG - Your organization's URL slug
+#   TRUNK_API_TOKEN - Your API token for authentication
+#
+# Optional environment variables for repository metadata:
+#   TRUNK_REPO_ROOT - Path to repository root
+#   TRUNK_REPO_URL - Repository URL (e.g., https://github.com/org/repo.git)
+#   TRUNK_REPO_HEAD_SHA - HEAD commit SHA
+#   TRUNK_REPO_HEAD_BRANCH - HEAD branch name
+#   TRUNK_REPO_HEAD_COMMIT_EPOCH - HEAD commit timestamp (seconds since epoch)
+#   TRUNK_REPO_HEAD_AUTHOR_NAME - HEAD commit author name
+#   TRUNK_PR_NUMBER - PR number, if uploading from a PR (normally inferred from CI environment variables)
+#
+# Optional environment variables for configuration:
+#   TRUNK_CODEOWNERS_PATH - Path to CODEOWNERS file
+#   TRUNK_VARIANT - Variant name for test results (e.g., 'linux', 'pr-123')
+#   TRUNK_DISABLE_QUARANTINING - Set to 'true' to disable quarantining
+#   TRUNK_ALLOW_EMPTY_TEST_RESULTS - Set to 'true' to allow empty results
+#   TRUNK_DRY_RUN - Set to 'true' to save bundle locally instead of uploading
+#   TRUNK_USE_UNCLONED_REPO - Set to 'true' for uncloned repo mode
+#   TRUNK_LOCAL_UPLOAD_DIR - Directory to save test results locally (disables upload)
+#   TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS - Time to cache quarantined tests on disk (in seconds)
+#   DISABLE_RSPEC_TRUNK_FLAKY_TESTS - Set to 'true' to completely disable Trunk
+#
+require 'rspec/core'
+require 'time'
+require 'rspec_trunk_flaky_tests'
+
+# String is an override to the main String class that is used to colorize the output
+# it is used to make the output more readable
+class String
+  def colorize(color_code)
+    "\e[#{color_code}m#{self}\e[0m"
+  end
+
+  def red
+    colorize(31)
+  end
+
+  def green
+    colorize(32)
+  end
+
+  def yellow
+    colorize(33)
+  end
+end
+
+def escape(str)
+  str.dump[1..-2]
+end
+
+# Knapsack example detector instantiates all test cases in order to determine how to shard them
+# These instantiations should not generate test bundles, so we
+# disable the gem when running under knapsack_pro:rspec_test_example_detector
+def knapsack_detector_mode?
+  knapsack_detector_command?
+end
+
+def knapsack_detector_command?
+  command_line = "#{$PROGRAM_NAME} #{ARGV.join(' ')}".strip
+  command_line.include?('knapsack_pro:rspec_test_example_detector') ||
+    command_line.include?('knapsack_pro:queue:rspec:initialize')
+end
+
+def trunk_disabled
+  knapsack_detector_mode? || ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' ||
+    ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil?
+end
+
+# we want to cache the test report in memory so we can add to it as we go and reduce the number of API calls
+$test_report = TestReport.new('rspec', "#{$PROGRAM_NAME} #{ARGV.join(' ')}", nil)
+$failure_encountered_and_quarantining_disabled = false
+
+module RSpec
+  module Core
+    # Example is the class that represents a test case
+    class Example
+      # keep the original method around so we can call it
+      alias set_exception_core set_exception
+      alias assign_generated_description_core assign_generated_description
+      # RSpec uses the existance of an exception to determine if the test failed
+      # We need to override this to allow us to capture the exception and then
+      # decide if we want to fail the test or not
+      # trunk-ignore(rubocop/Naming/AccessorMethodName)
+      def set_exception(exception)
+        return set_exception_core(exception) if metadata[:pending]
+        return set_exception_core(exception) if trunk_disabled
+        return set_exception_core(exception) if metadata[:retry_attempts]&.positive?
+
+        handle_quarantine_check(exception)
+      end
+
+      # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength)
+      def handle_quarantine_check(exception)
+        id = generate_trunk_id
+        name = full_description
+        parent_name = example_group.metadata[:description]
+        parent_name = parent_name.empty? ? 'rspec' : parent_name
+        file = escape(metadata[:file_path])
+        classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
+        unless $failure_encountered_and_quarantining_disabled
+          puts "Test failed, checking if it can be quarantined: `#{location}`".yellow
+        end
+        is_quarantined_result = $test_report.is_quarantined(id, name, parent_name, classname, file)
+        if is_quarantined_result.quarantining_disabled_for_repo
+          unless $failure_encountered_and_quarantining_disabled
+            puts 'Quarantining is disabled for this repo, no failures will be quarantined'.yellow
+            $failure_encountered_and_quarantining_disabled = true
+          end
+          set_exception_core(exception)
+        elsif is_quarantined_result.test_is_quarantined
+          # monitor the override in the metadata
+          metadata[:quarantined_exception] = exception
+          puts "Test is quarantined, overriding exception: #{exception}".green
+          nil
+        else
+          puts 'Test is not quarantined, continuing'.red
+          set_exception_core(exception)
+        end
+      end
+
+      def assign_generated_description
+        metadata[:is_description_generated] = description_generated?
+        assign_generated_description_core
+      end
+
+      def description_generated?
+        return metadata[:is_description_generated] unless metadata[:is_description_generated].nil?
+
+        description == location_description
+      end
+
+      def generate_trunk_id
+        return "trunk:#{id}-#{location}" if description_generated?
+      end
+
+      # Procsy is a class that is used to wrap execution of the Example class
+      class Procsy
+        def run_with_trunk
+          RSpec::Trunk.new(self).run
+        end
+      end
+    end
+  end
+end
+
+module RSpec
+  # Trunk is a class that is used to monitor the execution of the Example class
+  class Trunk
+    def self.setup
+      RSpec.configure do |config|
+        if trunk_disabled
+          config.around(:each, &:run)
+        else
+          config.around(:each, &:run_with_trunk)
+        end
+      end
+    end
+
+    def initialize(example)
+      @example = example
+    end
+
+    def current_example
+      @current_example ||= RSpec.current_example
+    end
+
+    def run
+      # run the test
+      @example.run
+      # monitor attempts in the metadata
+      if @example.metadata[:attempt_number]
+        @example.metadata[:attempt_number] += 1
+      else
+        @example.metadata[:attempt_number] = 0
+      end
+    end
+  end
+end
+
+# TrunkAnalyticsListener is a class that is used to listen to the execution of the Example class
+# it generates and submits the final test reports
+class TrunkAnalyticsListener
+  def initialize
+    @testreport = $test_report
+  end
+
+  def example_finished(notification)
+    add_test_case(notification.example)
+  end
+
+  # trunk-ignore(rubocop/Metrics/MethodLength)
+  def close(_notification)
+    if $failure_encountered_and_quarantining_disabled
+      puts 'Note: Quarantining is disabled for this repo. Test failures were not quarantined.'.yellow
+    end
+    if ENV['TRUNK_LOCAL_UPLOAD_DIR']
+      saved = @testreport.try_save(ENV['TRUNK_LOCAL_UPLOAD_DIR'])
+      if saved
+        puts 'Local Flaky tests report generated'.green
+      else
+        puts 'Failed to generate local flaky tests report'.red
+      end
+    else
+      published = @testreport.publish
+      if published
+        puts 'Flaky tests report upload complete'.green
+      else
+        puts 'Failed to publish flaky tests report'.red
+      end
+    end
+  end
+
+  # trunk-ignore(rubocop/Metrics/CyclomaticComplexity,rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength)
+  def add_test_case(example)
+    failure_message = example.exception.to_s if example.exception
+    failure_message = example.metadata[:quarantined_exception].to_s if example.metadata[:quarantined_exception]
+    # TODO: should we use concatenated string or alias when auto-generated description?
+    name = example.full_description
+    file = escape(example.metadata[:file_path])
+    classname = file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')
+    line = example.metadata[:line_number]
+    started_at = example.execution_result.started_at.to_i
+    finished_at = example.execution_result.finished_at.to_i
+    id = example.generate_trunk_id
+
+    attempt_number = example.metadata[:retry_attempts] || example.metadata[:attempt_number] || 0
+    status = example.execution_result.status.to_s
+    # set the status to failure, but mark it as quarantined
+    is_quarantined = example.metadata[:quarantined_exception] ? true : false
+    case example.execution_result.status
+    when :passed
+      status = is_quarantined ? Status.new('failure') : Status.new('success')
+    when :failed
+      status = Status.new('failure')
+    when :pending
+      status = Status.new('skipped')
+    end
+    parent_name = example.example_group.metadata[:description]
+    parent_name = parent_name.empty? ? 'rspec' : parent_name
+    @testreport.add_test(id, name, classname, file, parent_name, line, status, attempt_number,
+                         started_at, finished_at, failure_message || '', is_quarantined)
+  end
+end
+
+RSpec.configure do |c|
+  next if trunk_disabled
+
+  c.reporter.register_listener TrunkAnalyticsListener.new, :example_finished, :close
+end
+
+RSpec::Trunk.setup

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: rspec_trunk_flaky_tests
+version: !ruby/object:Gem::Version
+  version: 0.12.9.pre.beta.0
+platform: ruby
+authors:
+- Trunk Technologies, Inc.
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Integrates RSpec with Trunk Flaky Tests to automatically upload test
+  results from your CI jobs. Enables accurate flaky test detection, quarantining,
+  and analytics.
+email: support@trunk.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/rspec_trunk_flaky_tests.rb
+- lib/trunk_spec_helper.rb
+homepage: https://docs.trunk.io/flaky-tests/get-started/frameworks/rspec
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/trunk-io/analytics-cli/tree/main/rspec-trunk-flaky-tests
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: RSpec plugin for Trunk Flaky Tests - automatically uploads test results to
+  detect and quarantine flaky tests
+test_files: []