ci_runner 0.1.0

data/lib/ci_runner/github_client.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "json"
+require "open-uri"
+
+module CIRunner
+  # A simple client to interact the GitHub API.
+  #
+  # @example Using the client
+  #  GithubClient.new("access_token").me
+  class GithubClient
+    Error = Class.new(StandardError)
+
+    # @return [Net::HTTP] An instance of Net:HTTP configured to make requests to the GitHub API endpoint.
+    def self.default_client
+      Net::HTTP.new("api.github.com", 443).tap do |http|
+        http.use_ssl = true
+        http.read_timeout = 3
+        http.write_timeout = 3
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      end
+    end
+
+    # @param access_token [String] The access token with "repo" scope.
+    # @param client [Net::HTTP]
+    def initialize(access_token, client = self.class.default_client)
+      @access_token = access_token
+      @client = client
+    end
+
+    # Make an API request to get the authenticated user. Used to verify if the access token
+    # the user has stored in its config is valid.
+    #
+    # @return [Hash] See GitHub documentation.
+    #
+    # @see https://docs.github.com/en/rest/users/users#get-the-authenticated-user
+    def me
+      get("/user")
+    end
+
+    # Makes an API request to get the CI checks for the +commit+.
+    #
+    # @param repository [String] The full repository name, including the owner (rails/rails)
+    # @param commit [String] The Git commit that has been pushed to GitHub.
+    #
+    # @return [Hash] See GitHub documentation.
+    #
+    # @see https://docs.github.com/en/rest/checks/runs#list-check-runs-for-a-git-reference
+    def check_runs(repository, commit)
+      get("/repos/#{repository}/commits/#{commit}/check-runs")
+    end
+
+    # Makes two requests to get the CI log for a check run.
+    # The first request returns a 302 containing a Location header poiting to a short lived url to download the log.
+    # The second request is to actually download the log.
+    #
+    # @param repository [String] The full repository name, including the owner (rails/rails)
+    # @param check_run_id [Integer] The GitHub ID of the check run.
+    #
+    # @return [Tempfile, IO] Depending on the size of the response. Quirk of URI.open.
+    #
+    # @see https://docs.github.com/en/rest/actions/workflow-jobs#download-job-logs-for-a-workflow-run
+    def download_log(repository, check_run_id)
+      download_url = get("/repos/#{repository}/actions/jobs/#{check_run_id}/logs")
+
+      URI.open(download_url) # rubocop:disable Security/Open
+    end
+
+    private
+
+    # Perform an authenticated GET request.
+    #
+    # @param path [String] The resource to access.
+    #
+    # @return (See #request)
+    def get(path)
+      request(Net::HTTP::Get, path)
+    end
+
+    # Perform an authenticated request.
+    #
+    # @param verb_class [Net::HTTPRequest] A subclass of Net::HTTPRequest.
+    # @param path [String] The resource to access.
+    #
+    # @return [Hash, String] A decoded JSON response or a String pointing to the Location redirection.
+    def request(verb_class, path)
+      req = verb_class.new(path)
+      req["Accept"] = "application/vnd.github+json"
+      req.basic_auth("user", @access_token)
+
+      response = @client.request(req)
+
+      case response.code.to_i
+      when 200..204
+        response.content_type == "application/json" ? JSON.parse(response.body) : response.body
+      when 302
+        response["Location"]
+      else
+        raise(Error, "GitHub response: Status: #{response.code}. Body:\n\n#{response.body}")
+      end
+    end
+  end
+end

data/lib/ci_runner/log_downloader.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "tmpdir"
+require "fileutils"
+
+module CIRunner
+  # A PORO to help download and cache a GitHub CI log.
+  #
+  # @example Using the service
+  #   log_dl = LogDownloader.new("commit_sha", "catanacorp/catana", { "id" => 1, "name" => "Ruby Test 3.1.2" })
+  #   log_file = log_dl.fetch
+  #   puts log_file # => File
+  #
+  # @see https://docs.github.com/en/rest/actions/workflow-jobs#download-job-logs-for-a-workflow-run
+  class LogDownloader
+    # @param commit [String] A Git commit. Used to compute the file name we are going to cache.
+    # @param repository [String] The repository full name, including the owner (i.e. rails/rails).
+    # @param check_run [Hash] A GitHub CI check for which we want to download the log.
+    def initialize(commit, repository, check_run)
+      @commit = commit
+      @repository = repository
+      @check_run = check_run
+    end
+
+    # Download the CI logs from GitHub or retrieve it from disk in case we previously downloaded it.
+    #
+    # @param block [Proc, Lambda] A proc that gets called if fetching the logs from GitHub fails. Allows the CLI to
+    #   prematurely exit while cleaning up the CLI::UI frame.
+    #
+    # @return [File] A file ready to be read.
+    def fetch(&block)
+      return cached_log if cached_log
+
+      github_client = GithubClient.new(Configuration::User.instance.github_token)
+      error = nil
+
+      ::CLI::UI.spinner("Downloading CI logs from GitHub", auto_debrief: false) do
+        logfile = github_client.download_log(@repository, @check_run["id"])
+
+        cache_log(logfile)
+      rescue GithubClient::Error => e
+        error = e
+
+        ::CLI::UI::Spinner::TASK_FAILED
+      end
+
+      block.call(error) if error
+      cached_log
+    end
+
+    private
+
+    # Store the log on the user's disk.
+    #
+    # @param logfile [Tempfile, IO] Depending on the size of the response. A quirk of URI.open.
+    #
+    # @return [void]
+    def cache_log(logfile)
+      FileUtils.mkdir_p(computed_file_path.dirname)
+
+      if logfile.is_a?(Tempfile)
+        FileUtils.cp(logfile, computed_file_path)
+      else
+        File.write(computed_file_path, logfile.read)
+      end
+    end
+
+    # @return [String] A path where to store the logfile on the users' disk.
+    #   The path is composed of the commit, the CI check name and the repository full name.
+    #
+    # @return [Pathname]
+    #
+    # @example Given a repository "rails/rails". A CI check called "Ruby 3.0". A commit "abcdef".
+    #   puts computed_filed_path # ==> /var/tmpdir/T/.../rails/rails/log-abcdef-Ruby 3.0
+    def computed_file_path
+      normalized_run_name = @check_run["name"].tr("/", "_")
+
+      log_folder.join("log-#{@commit[0..12]}-#{normalized_run_name}.log")
+    end
+
+    # @return [Pathname]
+    def log_folder
+      Pathname(Dir.tmpdir).join(@repository)
+    end
+
+    # @return [Pathname, false] Depending if the log has been downloaded before.
+    def cached_log
+      return false unless computed_file_path.exist?
+
+      computed_file_path
+    end
+  end
+end

data/lib/ci_runner/runners/base.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "open3"
+
+module CIRunner
+  module Runners
+    class Base
+      # @return [Array<TestFailure>]
+      attr_accessor :failures
+
+      # @return (See TestFailure#seed)
+      attr_accessor :seed
+
+      # @return [String] The ruby version detected.
+      attr_accessor :ruby_version
+
+      # @return [String] The Gemfile detected.
+      attr_accessor :gemfile
+
+      # Children needs to implement this method to tell if they recognize the log output and if it can process them.
+      #
+      # @param _ [String] The CI log output.
+      #
+      # @return [Boolean]
+      def self.match?(_)
+        raise NotImplementedError, "Subclass responsability"
+      end
+
+      # @param ci_log [String] The CI log output.
+      def initialize(ci_log)
+        @ci_log = ci_log
+        @failures = []
+        @buffer = +""
+      end
+
+      # Parse the CI log. Iterate over each line and try to detect:
+      #
+      # - The Ruby version
+      # - The Gemfile used
+      # - Failures (Including their names, their class and the file path)
+      #
+      # @return [void]
+      def parse!
+        @ci_log.each_line do |line|
+          case line
+          when seed_regex
+            @seed = first_matching_group(Regexp.last_match)
+          when ruby_detection_regex
+            @ruby_version = first_matching_group(Regexp.last_match)
+
+            @buffer << line if buffering?
+          when gemfile_detection_regex
+            @gemfile = first_matching_group(Regexp.last_match)
+          when buffer_detection_regex
+            if Configuration::Project.instance.process_on_new_match? && buffering?
+              process_buffer
+              @buffer.clear
+            end
+
+            @buffer << line
+          else
+            @buffer << line if buffering?
+          end
+        end
+
+        process_buffer if buffering?
+      end
+
+      # Entrypoint to start the runner process once it finishes parsing the log.
+      #
+      # @return [Void]
+      def start!
+        if ruby_version && !ruby_path.exist?
+          ::CLI::UI.puts(<<~EOM)
+            {{warning:Couldn't find Ruby version #{ruby_version} on your system.}}
+            {{warning:Searched in #{ruby_path}}}
+
+            {{warning:The test run will start but will be running using your current Ruby version {{underline:#{RUBY_VERSION}}}.}}
+          EOM
+        end
+
+        if gemfile && !gemfile_path.exist?
+          ::CLI::UI.puts(<<~EOM)
+            {{warning:Your CI run ran with the Gemfile #{gemfile}}}
+            {{warning:I couldn't find this gemfile in your folder.}}
+
+            {{warning:The test run will start but will be using the default Gemfile of your project}}
+          EOM
+        end
+      end
+
+      # Output useful information to the user before the test starts. This can only be called after the runner finished
+      # parsing the log.
+      #
+      # @return [void]
+      def report
+        default_ruby = "No specific Ruby version detected. Will be using your current version #{RUBY_VERSION}"
+        using_ruby = ruby_version ? ruby_version : default_ruby
+
+        default_gemfile = "No specific Gemfile detected. Will be using the default Gemfile of your project."
+        using_gemfile = gemfile ? gemfile : default_gemfile
+
+        ::CLI::UI.puts(<<~EOM)
+
+          - Test framework detected:    {{info:#{name}}}
+          - Detected Ruby version:      {{info:#{using_ruby}}}
+          - Detected Gemfile:           {{info:#{using_gemfile}}}
+          - Number of failings tests:   {{info:#{failures.count}}}
+        EOM
+      end
+
+      private
+
+      # Process the +@buffer+ to find any test failures. Uses the project's regex if set, or fallbacks to
+      # the default set of regexes this gem provides.
+      #
+      # See Project#buffer_starts_regex for explanation on the difference between the buffer and the CI log output.
+      #
+      # @return [void]
+      def process_buffer
+        custom_project_regex = Configuration::Project.instance.test_failure_detection_regex
+
+        if custom_project_regex
+          custom_project_regex.match(@buffer) do |match_data|
+            @failures << TestFailure.new(match_data[:class], match_data[:test_name], match_data[:file_path])
+          end
+        else
+          yield
+        end
+      end
+
+      # See Configuration::Project#ruby_detection_regex
+      #
+      # @return [Regexp]
+      def ruby_detection_regex
+        return @ruby_detection_regex if defined?(@ruby_detection_regex)
+
+        regexes = [
+          Configuration::Project.instance.ruby_detection_regex,
+          %r{[^_-][rR]uby(?:[[:blank:]]*|/)(\d\.\d\.\d+)p?(?!/gems)},
+        ].compact
+
+        @ruby_detection_regex = Regexp.union(*regexes)
+      end
+
+      # See Configuration::Project#gemfile_detection_regex
+      #
+      # @return [Regexp]
+      def gemfile_detection_regex
+        return @gemfile_detection_regex if defined?(@gemfile_detection_regex)
+
+        regexes = [
+          Configuration::Project.instance.gemfile_detection_regex,
+          /BUNDLE_GEMFILE:[[:blank:]]*(.*)/,
+        ].compact
+
+        @gemfile_detection_regex = Regexp.union(*regexes)
+      end
+
+      # See Configuration::Project#seed_detection_regex
+      #
+      # @return [Regexp]
+      def seed_regex
+        return @seed_regex if defined?(@seed_regex)
+
+        regexes = [
+          Configuration::Project.instance.seed_detection_regex,
+          self.class::SEED_REGEX,
+        ].compact
+
+        @seed_regex = Regexp.union(*regexes)
+      end
+
+      # See Configuration::Project#buffer_starts_regex
+      #
+      # @return [Regexp]
+      def buffer_detection_regex
+        return @buffer_detection_regex if defined?(@buffer_detection_regex)
+
+        regexes = [
+          Configuration::Project.instance.buffer_starts_regex,
+          self.class::BUFFER_STARTS,
+        ].compact
+
+        @buffer_detection_regex = Regexp.union(*regexes)
+      end
+
+      # @return [Pathname, nil] The absolute path of the detected Gemfile based on where the user ran
+      #   the `ci_runner` command from. Nil in no Gemfile was detected during parsing.
+      def gemfile_path
+        return unless gemfile
+
+        Pathname(Dir.pwd).join(gemfile)
+      end
+
+      # @return [Pathname, nil] The absolute path of the Ruby binary on the user's machine.
+      #   Nil if no Ruby version was detected when parsing.
+      #
+      # @return [Pathname]
+      def ruby_path
+        return unless ruby_version
+
+        Pathname(Dir.home).join(".rubies/ruby-#{ruby_version}/bin/ruby")
+      end
+
+      # @return [Boolean]
+      def buffering?
+        !@buffer.empty?
+      end
+
+      # Regexp#union with capturing groups makes it difficult to know which subregex matched
+      # and therefore which group to get. Convenient method to get the first whatever value is non nil.
+      # There should be only one truty value in all groups.
+      #
+      # @param match_data [MatchData]
+      #
+      # @return [String]
+      def first_matching_group(match_data)
+        match_data.captures.find { |v| v }&.rstrip
+      end
+
+      # Runs a command and stream its output. We can't use `system` directly, as otherwise the
+      # streamed ouput won't fit properly inside the ::CLI::UI frame.
+      #
+      # @param env [Hash] A hash of environment variables to pass to the subprocess
+      # @param command [String] The command itself
+      #
+      # @return [void]
+      def execute_within_frame(env, command)
+        Open3.popen3(env, command) do |_, stdout, stderr, _|
+          while (char = stdout.getc)
+            print(char)
+          end
+
+          print(stderr.read)
+        end
+      end
+    end
+  end
+end

data/lib/ci_runner/runners/minitest_runner.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require "drb/drb"
+require "tempfile"
+require "rake"
+
+module CIRunner
+  module Runners
+    # Runner responsible to detect parse and process a CI output log generated by Minitest.
+    #
+    # Because minitest doesn't have a CLI built-in, there is a lot of complications to re-run
+    # a selection of tests, especially when we need to run the tests in a subprocess.
+    #
+    # In a nutshell, this Runner will try its best to get the file path of each failures.
+    # Without a custom repoter, Minitest will fail pointing to the right file a lot of the time.
+    # Therefore CI Runner tries a mix of possibilities (by looking at the stack trace, inferring the class name).
+    #
+    # Once the logs have been parsed, we tell Ruby to load only the test files that failed. By default, loading
+    # those files would make Minitest run all tests included in those files, where we want to run only tests
+    # that failed on CI.
+    #
+    # Minitest doesn't have a way to filter by test name (the -n) isn't powerful enough as two test suites can
+    # contain the same name.
+    # CI Runner launches a DRB server over a UNIX socket which allows allows the subprocess running minitest
+    # to know whether a test should ran. This is accomplished in combination with a Minitest plugin.
+    #
+    # A vanilla rake task plugs the whole thing in order to not reinvent the wheel.
+    class MinitestRunner < Base
+      SEED_REGEX = Regexp.union(
+        /Run options:.*?--seed\s+(\d+)/, # Default Minitest Statistics Repoter
+        /Running tests with run options.*--seed\s+(\d+)/, # MinitestReporters BaseReporter
+        /Started with run options.*--seed\s+(\d+)/, # MinitestReporters ProgressReporter
+      )
+      BUFFER_STARTS = /(Failure|Error):\s*\Z/
+
+      # @param ci_log [String] The CI log output
+      #
+      # @return [Boolean] Whether this runner detects (and therefore can handle) Minitest from the log output.
+      def self.match?(ci_log)
+        default_reporter = %r{(Finished in) \d+\.\d{6}s, \d+\.\d{4} runs/s, \d+\.\d{4} assertions/s\.}
+
+        Regexp.union(default_reporter, SEED_REGEX, "minitest").match?(ci_log)
+      end
+
+      # @return [String] See Runners::Base#report
+      def name
+        "Minitest"
+      end
+
+      # Start a subprocess to rerun the detected failing tests.
+      #
+      # Few things to note:
+      #
+      # - CI Runner is meant to be installed as a standalone gem, not Bundled (in a Gemfile). CI Runner
+      #   doesn't know what's inside the loaded specs of the application and it's possible (while unlikely)
+      #   that "Rake" isn't part of the application/gem dependencies. Therefore, when activativing Bundler,
+      #   requiring "rake/testtask" would fail inside the Rakefile.
+      #
+      #   To avoid this problem, requiring rake before Bundler gets activated (using the ruby -rswitch).
+      #
+      # - The CI Runner Minitest plugin will not be detected once the subprocess starts. (Again because CI Runner
+      #   is not part of the application dependencies). Adding it to the LOAD_PATH manually is required.
+      def start!
+        super
+
+        minitest_plugin_path = File.expand_path("../..", __dir__)
+        rake_load_path = Gem.loaded_specs["rake"].full_require_paths.first
+
+        code = <<~EOM
+          Rake::TestTask.new(:__ci_runner_test) do |t|
+            t.libs << "test"
+            t.libs << "lib"
+            t.libs << "#{rake_load_path}"
+            t.libs << "#{minitest_plugin_path}"
+            t.test_files = #{failures.map(&:path)}
+          end
+
+          Rake::Task[:__ci_runner_test].invoke
+        EOM
+
+        rakefile_path = File.expand_path("Rakefile", Dir.mktmpdir)
+        File.write(rakefile_path, code)
+
+        server = DRb.start_service("drbunix:", failures)
+
+        env = { "TESTOPTS" => "--ci-runner=#{server.uri}" }
+        env["SEED"] = seed if seed
+        env["RUBY"] = ruby_path.to_s if ruby_path&.exist?
+        env["BUNDLE_GEMFILE"] = gemfile_path.to_s if gemfile_path&.exist?
+
+        execute_within_frame(env, "bundle exec ruby -r'rake/testtask' #{rakefile_path}")
+
+        DRb.stop_service
+      end
+
+      private
+
+      def process_buffer
+        super do
+          match_data = minitest_failure
+          next unless match_data
+
+          file_path = valid_path?(match_data[:file_path]) ? match_data[:file_path] : find_test_location(match_data)
+
+          @failures << TestFailure.new(match_data[:class], match_data[:test_name], file_path)
+        end
+      end
+
+      # Detect if the buffer contains a Minitest Error/Failure. If not, bail out.
+      #
+      # @return [nil, MatchData] Whether the regex matched.
+      def minitest_failure
+        regex = /(?:\s*)(?<class>[a-zA-Z0-9_:]+)\#(?<test_name>test_.+?)(:\s*$|\s+\[(?<file_path>.*):\d+\])/
+
+        regex.match(@buffer)
+      end
+
+      # There are two different type of errors in Minitest: Failure and Error.
+      # A failure is an assertion that failed `assert(false)`. An error in an unexpected exception `foo.boom`.
+      #
+      # For failure, Minitest will print the location of the test file, but it can be wrong.
+      #
+      # MaintenanceTasks::RunsTest#test_run_a_CSV_Task [/home/runner/work/maintenance_tasks/maintenance_tasks/vendor/bundle/ruby/2.7.0/gems/capybara-3.37.1/lib/capybara/minitest.rb:295] # rubocop:disable Layout/LineLength
+      #
+      # In this case the location of the file points to a file inside a gem, which is for sure not where the test lives.
+      # When this happen, we discard the location provided by Minitest and try to match another possible location.
+      #
+      # @param path [String] A path to a file
+      #
+      # @return [Boolean] Whether the path points to a gem.
+      def valid_path?(path)
+        return false if path.nil?
+
+        points_to_a_gem = %r{ruby/.*?/gems}
+
+        !path.match?(points_to_a_gem)
+      end
+
+      # Try different alternatives to find where the test file lives.
+      #
+      # @param match_data [MatchData] The match from the +minitest_failures+
+      #
+      # @return [String] The path of the test file.
+      def find_test_location(match_data)
+        match = try_rails
+        return match if match
+
+        match = try_infer_file_from_class(match_data)
+        return match if match
+
+        match = try_stacktrace(match_data)
+        return match if match
+
+        raise("Can't find test location")
+      end
+
+      # Taken from ActiveSupport
+      #
+      # @param camel_cased_word [String]
+      #
+      # @return [String]
+      def underscore(camel_cased_word)
+        return camel_cased_word.to_s unless /[A-Z-]|::/.match?(camel_cased_word)
+
+        word = camel_cased_word.to_s.gsub("::", "/")
+        word.gsub!(/([A-Z]+)(?=[A-Z][a-z])|([a-z\d])(?=[A-Z])/) do
+          (Regexp.last_match(1) || Regexp.last_match(2)) << "_"
+        end
+        word.tr!("-", "_")
+        word.downcase!
+        word
+      end
+
+      # Try to find the file location using the class name we matched.
+      #
+      # @param match_data [MatchData] The match from the +minitest_failures+
+      #
+      # @return [String, nil] The path of the test file. Nil if not found.
+      #
+      # @example
+      #   Given a failure: TestReloading#test_reload_recovers_from_name_errors__w__on_unload_callbacks_:
+      #   We try to look for a line from the stacktrace like this one:
+      #   /Users/runner/wok/zeitwerk/zeitwerk/test/lib/zeitwerk/test_reloading.rb:12:in `block in <class:TestReloading>'
+      def try_stacktrace(match_data)
+        regex = %r{\s*(/.*?):\d+:in.*#{match_data[:class]}}
+
+        @buffer.match(regex) { |match| match[1] }
+      end
+
+      # Try to find the file location using the class_name -> file convention naming
+      #
+      # @param match_data [MatchData] The match from the +minitest_failures+
+      #
+      # @return [String, nil] The path of the test file. Nil if not found.
+      #
+      # @example
+      #   Given a failure: BigProblemTest#test_this_is_problematic
+      #
+      #   We try to look for a line from the stacktrace like this one:
+      #   /Users/runner/work/foo/foo/test/lib/foo/big_problem_test.rb:218
+      def try_infer_file_from_class(match_data)
+        file_name = underscore(match_data[:class].split("::").last)
+        regex = %r{(/.*#{file_name}.*?):\d+}
+
+        @buffer.match(regex) { |match| match[1] }
+      end
+
+      # Try to find the file location using the Rails reporter
+      #
+      # @param match_data [MatchData] The match from the +minitest_failures+
+      #
+      # @return [String, nil] The path of the test file. Nil if not found.
+      def try_rails
+        regex = /rails\s+test\s+(.*?):\d+/
+
+        @buffer.match(regex) { |match| match[1] }
+      end
+    end
+  end
+end