rspecq-instructure 0.8.1

data/lib/rspecq/reporter.rb

@@ -0,0 +1,195 @@
+module RSpecQ
+  # A Reporter, given a build ID, is responsible for consolidating the results
+  # from different workers and printing a complete build summary to the user,
+  # along with any failures that might have occurred.
+  #
+  # The failures are printed in real-time as they occur, while the final
+  # summary is printed after the queue is empty and no tests are being
+  # executed. If the build failed, the status code of the reporter is non-zero.
+  #
+  # Reporters are readers of the queue.
+  class Reporter
+    def initialize(build_id:, timeout:, redis_opts:, worker_liveness_sec:, queue_wait_timeout: 30,
+                   update_timings: false, timings_key: nil)
+      @build_id = build_id
+      @timeout = timeout
+      @queue = Queue.new(build_id, "reporter", redis_opts, worker_liveness_sec)
+      @queue_wait_timeout = queue_wait_timeout
+      @update_timings = update_timings
+      @timings_key = timings_key
+
+      # We want feedback to be immediately printed to CI users, so
+      # we disable buffering.
+      $stdout.sync = true
+    end
+
+    def report
+      @queue.wait_until_published(@queue_wait_timeout)
+
+      finished = false
+
+      reported_failures = {}
+      failure_heading_printed = false
+
+      @timeout.times do
+        @queue.example_failures.each do |job, rspec_output|
+          next if reported_failures[job]
+
+          if !failure_heading_printed
+            puts "\nFailures:\n"
+            failure_heading_printed = true
+          end
+
+          reported_failures[job] = true
+          puts failure_formatted(rspec_output)
+        end
+
+        unless @queue.exhausted? || @queue.build_failed_fast?
+          sleep 1
+          next
+        end
+
+        finished = true
+        break
+      end
+
+      raise "Build not finished after #{@timeout} seconds" if !finished
+
+      # The reporter can observe the build finished before any worker stamps
+      # finished_at; stamp it here (setnx, first writer wins) so the build
+      # duration — and our canvas-consumed key_build_time — are always recorded.
+      @queue.try_mark_finished
+
+      build_duration = test_durations&.first
+      @queue.record_build_time(build_duration) if build_duration
+
+      if @update_timings && @queue.build_successful?
+        if @timings_key
+          puts "Updating job timings @ #{@timings_key}"
+          @queue.update_global_timings(@timings_key)
+        else
+          puts "Updating global job timings"
+          @queue.update_global_timings
+        end
+      end
+
+      flaky_jobs = @queue.flaky_jobs
+
+      puts summary(@queue.example_failures, @queue.non_example_errors, flaky_jobs)
+
+      flaky_jobs_to_sentry(flaky_jobs, build_duration)
+
+      exit 1 if !@queue.build_successful?
+    end
+
+    private
+
+    # Two build durations (secs): from master election, and from queue ready.
+    # nil until the build has both a start and a finish timestamp.
+    def test_durations
+      @test_durations ||= @queue.took_times_secs
+    end
+
+    # We try to keep this output consistent with RSpec's original output
+    def summary(failures, errors, flaky_jobs)
+      failed_examples_section = "\nFailed examples:\n\n"
+
+      failures.each_value do |msg|
+        parts = msg.split("\n")
+        failed_examples_section << "  #{parts[-1]}\n"
+      end
+
+      summary = ""
+      if @queue.build_failed_fast?
+        summary << "\n\n"
+        summary << "The limit of #{@queue.fail_fast} failures has been reached\n"
+        summary << "Aborting..."
+        summary << "\n"
+      end
+
+      summary << failed_examples_section if !failures.empty?
+
+      errors.each_value { |msg| summary << msg }
+
+      requeues = @queue.requeued_jobs.values.sum
+
+      summary << "\n"
+      summary << "Total results:\n"
+      summary << "  #{@queue.example_count} examples " \
+                 "(#{@queue.processed_jobs_count} jobs), " \
+                 "#{failures.count} failures, " \
+                 "#{errors.count} errors, " \
+                 "#{requeues} requeues"
+      summary << ", #{flaky_jobs.count} flaky" if flaky_jobs.any?
+      summary << ", #{@queue.lost_jobs_count} lost jobs (unique)" if @queue.lost_jobs_count.positive?
+      summary << "\n\n\n"
+
+      from_elected_master, from_queue_ready = test_durations
+      if from_elected_master
+        summary << "Spec time (from elected master): #{humanize_duration(from_elected_master)}\n"
+      end
+      if from_queue_ready
+        summary << "Spec time (from queue ready): #{humanize_duration(from_queue_ready)}\n"
+      end
+      summary << "Worker total execution time: " \
+                 "#{humanize_duration(@queue.total_execution_time_ms / 1000)}"
+
+      if !flaky_jobs.empty?
+        summary << "\n\n"
+        summary << "Flaky jobs detected (count=#{flaky_jobs.count}):\n"
+        flaky_jobs.each do |j|
+          job_timing = (jt = @queue.job_build_timing(j)) ? humanize_duration(jt.to_i) : "---"
+          summary << RSpec::Core::Formatters::ConsoleCodes.wrap(
+            "#{@queue.job_location(j)} @ #{@queue.failed_job_worker(j)} timing=#{job_timing}\n",
+            RSpec.configuration.pending_color
+          )
+
+          next if ENV["RSPECQ_REPORTER_RERUN_COMMAND_SKIP"]
+
+          summary << "#{@queue.job_rerun_command(j)}\n\n\n"
+        end
+      end
+
+      summary
+    end
+
+    def failure_formatted(rspec_output)
+      rspec_output.split("\n")[0..-2].join("\n")
+    end
+
+    def humanize_duration(secs)
+      min, sec = secs.divmod(60)
+
+      format("%<min>d:%<sec>02d", min: min, sec: sec)
+    end
+
+    def flaky_jobs_to_sentry(jobs, build_duration)
+      return if jobs.empty?
+
+      jobs.each do |job|
+        filename = job.gsub(%r{\[.+\]|\./}, "").split(":")[0]
+
+        extra = {
+          build: @build_id,
+          build_timeout: @timeout,
+          build_duration: build_duration,
+          location: @queue.job_location(job),
+          rerun_command: @queue.job_rerun_command(job),
+          worker: @queue.failed_job_worker(job)
+        }
+
+        tags = {
+          flaky: true,
+          spec_file: filename
+        }
+
+        Sentry.capture_message(
+          "Flaky test in #{filename}",
+          level: "warning",
+          extra: extra,
+          tags: tags
+        )
+      end
+    end
+  end
+end

data/lib/rspecq/version.rb

@@ -0,0 +1,3 @@
+module RSpecQ
+  VERSION = "0.8.1".freeze
+end

data/lib/rspecq/worker.rb

@@ -0,0 +1,418 @@
+require "json"
+require "pathname"
+require "pp"
+require "open3"
+
+module RSpecQ
+  # A Worker, given a build ID, continuously consumes tests off the
+  # corresponding and executes them, until the queue is empty.
+  # It is also responsible for populating the initial queue.
+  #
+  # Essentially, a worker is an RSpec runner that prints the results of the
+  # tests it executes to standard output.
+  #
+  # The typical use case is to spawn many workers for a given build, thereby
+  # parallelizing the work and achieving faster build times.
+  #
+  # Workers are readers+writers of the queue.
+  class Worker
+    # The root path or individual spec files to execute.
+    #
+    # Defaults to "spec" (similar to RSpec)
+    attr_accessor :files_or_dirs_to_run
+
+    # If set, spec files that are known to take more than this value to finish,
+    # will be split and scheduled on a per-example basis.
+    #
+    # Defaults to 999999
+    attr_accessor :file_split_threshold
+
+    # Retry failed examples up to N times (with N being the supplied value)
+    # before considering them legit failures
+    #
+    # Defaults to 3
+    attr_accessor :max_requeues
+
+    # Stop the execution after N failed tests. Do not stop at any point when
+    # set to 0.
+    #
+    # Defaults to 0
+    attr_accessor :fail_fast
+
+    # Time to wait for a queue to be published.
+    #
+    # Defaults to 30
+    attr_accessor :queue_wait_timeout
+
+    # The RSpec seed
+    attr_accessor :seed
+
+    # Reproduction flag. If true, worker will publish files in the exact order
+    # given in the command.
+    attr_accessor :reproduction
+
+    # Include a suite counter in any output filenames so that each suite run
+    # Output Junit formatted XML
+    # Output Junit formatted XML to a specified file
+    #
+    # Example: test_results/results-{{TEST_ENV_NUMBER}}-{{JOB_INDEX}}.xml
+    # where TEST_ENV_NUMBER is substituted with the environment variable
+    # from the gem parallel test, and JOB_INDEX is incremented based
+    # on the number of test suites run in the current process.
+    attr_accessor :junit_output
+
+    # Optional arguments to pass along to rspec.
+    #
+    # Defaults to nil
+    attr_accessor :rspec_args
+
+    # RSpec tags to filter examples by (e.g. "slow" or "~slow"). Repeatable.
+    #
+    # Defaults to []
+    attr_accessor :tags
+
+    # Target duration in seconds for time-balanced example chunks.
+    # When splitting slow files, examples are grouped into chunks of
+    # approximately this duration to reduce Kernel.load calls.
+    #
+    # Defaults to 30
+    attr_accessor :chunk_target_duration
+
+    attr_reader :queue, :build_id, :worker_id
+
+    def initialize(build_id:, worker_id:, redis_opts:, worker_liveness_sec:)
+      @build_id = build_id
+      @worker_id = worker_id
+      @worker_liveness_sec = worker_liveness_sec
+      @heartbeat_frequency = worker_liveness_sec / 6
+      @queue = Queue.new(build_id, worker_id, redis_opts, worker_liveness_sec)
+      @fail_fast = 0
+      @files_or_dirs_to_run = ["spec"]
+      @file_split_threshold = 999_999
+      @heartbeat_updated_at = nil
+      @max_requeues = 3
+      @queue_wait_timeout = 30
+      @seed = srand && (srand % 0xFFFF)
+      @reproduction = false
+      @tags = []
+      @junit_output = nil
+      @chunk_target_duration = 30
+
+      RSpec::Core::Formatters.register(Formatters::JobTimingRecorder, :dump_summary)
+      RSpec::Core::Formatters.register(Formatters::ExampleCountRecorder, :dump_summary)
+      RSpec::Core::Formatters.register(Formatters::FailureRecorder, :example_failed, :message)
+      RSpec::Core::Formatters.register(Formatters::WorkerHeartbeatRecorder, :example_finished)
+      RSpec::Core::Formatters.register(Formatters::JUnitFormatter, :example_passed, :example_failed,
+                                       :start, :stop, :dump_summary)
+    end
+
+    def work
+      puts "Working for build #{@build_id} (worker=#{@worker_id})"
+
+      # If this worker crashed previously (OOM kill, etc.), it may have a job
+      # stuck in the running hash. Recover it before doing anything else —
+      # otherwise reserve_job will silently overwrite it.
+      recovered = queue.recover_own_job
+      puts "Recovered abandoned job from previous crash: #{recovered}" if recovered
+
+      q_size = try_publish_queue!(queue)
+      puts "Published queue (size=#{q_size})" if q_size
+      queue.wait_until_published(queue_wait_timeout)
+      queue.save_worker_seed(@worker_id, seed)
+
+      # Use `--seed` to deterministically reproduce test failures
+      # related to randomization by passing the same `--seed` value
+      # as the one that triggered the failure.
+      #
+      # We also use the same seed to feed Rspec's `--seed` option.
+      Kernel.srand(seed)
+
+      idx = 0
+      loop do
+        # we have to bootstrap this so that it can be used in the first call
+        # to `requeue_lost_job` inside the work loop
+        update_heartbeat
+
+        if queue.build_failed_fast?
+          queue.try_mark_finished
+          return
+        end
+
+        lost, lost_worker = queue.requeue_lost_job
+        puts "Requeued lost job: #{lost} from #{lost_worker}" if lost
+
+        # TODO: can we make `reserve_job` also act like exhausted? and get
+        # rid of `exhausted?` (i.e. return false if no jobs remain)
+        job = queue.reserve_job
+
+        # build is finished
+        if job.nil? && queue.exhausted?
+          queue.try_mark_finished
+          return
+        end
+
+        if job.nil?
+          # backoff if no job is available
+          sleep 1
+          next
+        end
+
+        puts
+        puts "Executing #{job}"
+
+        ENV["ERROR_CONTEXT_BASE_PATH"] = nil
+        unless queue.is_requeue(job).nil?
+          ENV["ERROR_CONTEXT_BASE_PATH"] = "/usr/src/app/log/spec_failures/Rerun_#{queue.is_requeue(job)}"
+        end
+
+        reset_rspec_state!
+
+        # reconfigure rspec
+        RSpec.configuration.detail_color = :magenta
+        RSpec.configuration.seed = seed
+        RSpec.configuration.backtrace_formatter.filter_gem("rspecq")
+        RSpec.configuration.add_formatter(Formatters::FailureRecorder.new(queue, job, max_requeues, @worker_id))
+
+        if junit_output
+          RSpec.configuration.add_formatter(Formatters::JUnitFormatter.new(queue, job, max_requeues,
+                                                                           idx, junit_output))
+        end
+
+        RSpec.configuration.add_formatter(Formatters::ExampleCountRecorder.new(queue))
+        RSpec.configuration.add_formatter(Formatters::WorkerHeartbeatRecorder.new(self))
+
+        # Recording is always-on: every build records per-job timings into the
+        # build-scoped key. The reporter promotes them to the global key only
+        # when --update-timings is set.
+        RSpec.configuration.add_formatter(Formatters::JobTimingRecorder.new(queue, job))
+
+        args = [*rspec_args, "--format", "progress", *job.split("+")]
+        tags.each { |tag| args.push("--tag", tag) }
+        opts = RSpec::Core::ConfigurationOptions.new(args)
+
+        _result = RSpec::Core::Runner.new(opts).run($stderr, $stdout)
+
+        queue.acknowledge_job(job)
+        idx += 1
+      end
+    end
+
+    # Update the worker heartbeat if necessary
+    def update_heartbeat
+      if @heartbeat_updated_at.nil? || elapsed(@heartbeat_updated_at) >= @heartbeat_frequency
+        queue.record_worker_heartbeat
+        @heartbeat_updated_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+
+    # Memoized global timings — read once and reused across the scheduling path.
+    def global_timings
+      @global_timings ||= queue.global_timings
+    end
+
+    def try_publish_queue!(queue)
+      return if !queue.become_master
+
+      queue.mark_elected_master_at
+
+      if reproduction
+        q_size = queue.publish(files_or_dirs_to_run, fail_fast)
+        log_event(
+          "Reproduction mode. Published queue as given (size=#{q_size})",
+          "info"
+        )
+        return q_size
+      end
+
+      puts "I am the master worker (worker_id=#{@worker_id}), publishing the queue..."
+
+      RSpec.configuration.files_or_directories_to_run = files_or_dirs_to_run
+      files_to_run = RSpec.configuration.files_to_run.map { |j| relative_path(j) }
+
+      if global_timings.empty?
+        q_size = queue.publish(files_to_run.shuffle, fail_fast)
+        log_event(
+          "No timings found! Published queue in random order (size=#{q_size})",
+          "warning"
+        )
+        return q_size
+      end
+
+      # prepare jobs to run
+      jobs = []
+      slow_files = []
+
+      if file_split_threshold
+        slow_files = global_timings.take_while do |_job, duration|
+          duration >= file_split_threshold
+        end.map(&:first) & files_to_run
+      end
+
+      if slow_files.any?
+        jobs.concat(files_to_run - slow_files)
+        example_ids = files_to_example_ids(slow_files)
+        chunks = build_time_balanced_chunks(example_ids, global_timings, chunk_target_duration)
+        jobs.concat(chunks)
+      else
+        jobs.concat(files_to_run)
+      end
+
+      jobs = order_jobs_by_timings(jobs)
+
+      queue.publish(jobs, fail_fast)
+    end
+
+    private
+
+    # Assign each job its previous timing (chunk jobs sum their examples;
+    # untimed jobs get the median so they land mid-queue), then order slowest
+    # first so the longest jobs start earliest.
+    def order_jobs_by_timings(jobs)
+      default_timing = global_timings.values[global_timings.values.size / 2]
+
+      jobs = jobs.each_with_object({}) do |j, h|
+        if j.include?("+")
+          parts = j.split("+")
+          h[j] = parts.sum { |p| global_timings[p] || default_timing }
+        else
+          puts "Untimed job: #{j}" if global_timings[j].nil?
+          h[j] = global_timings[j] || default_timing
+        end
+      end
+
+      jobs.sort_by { |_j, t| -t }.map(&:first)
+    end
+
+    # Groups example IDs into time-balanced chunks, one chunk per Kernel.load.
+    # Examples from different files are never mixed. Uses per-example timings
+    # from Redis when available; falls back to file_timing / example_count.
+    #
+    # Returns an array of job strings: single examples are bare IDs, multi-example
+    # chunks are "+"-delimited (e.g. "spec/foo.rb[1:1]+spec/foo.rb[1:2]").
+    def build_time_balanced_chunks(example_ids, timings, target_duration)
+      by_file = example_ids.group_by { |id| id.sub(/\[.*\]$/, "") }
+      chunks = []
+
+      by_file.each do |file, ids|
+        file_timing = timings[file]
+        default_example_timing = file_timing ? file_timing / ids.size : target_duration / 10.0
+
+        timed_ids = ids.map { |id| [id, timings[id] || default_example_timing] }
+        timed_ids.sort_by! { |_, t| -t }
+
+        current_chunk = []
+        current_duration = 0.0
+
+        timed_ids.each do |id, duration|
+          if current_chunk.empty? || current_duration + duration <= target_duration
+            current_chunk << id
+            current_duration += duration
+          else
+            chunks << current_chunk.join("+")
+            current_chunk = [id]
+            current_duration = duration
+          end
+        end
+        chunks << current_chunk.join("+") unless current_chunk.empty?
+      end
+
+      chunks
+    end
+
+    def reset_rspec_state!
+      RSpec.clear_examples
+
+      # see https://github.com/rspec/rspec-core/pull/2723
+      if Gem::Version.new(RSpec::Core::Version::STRING) <= Gem::Version.new("3.9.1")
+        RSpec.world.instance_variable_set(
+          :@example_group_counts_by_spec_file, Hash.new(0)
+        )
+      end
+
+      # RSpec.clear_examples does not reset those, which causes issues when
+      # a non-example error occurs (subsequent jobs are not executed)
+      # TODO: upstream
+      RSpec.world.non_example_failure = false
+
+      # we don't want an error that occured outside of the examples (which
+      # would set this to `true`) to stop the worker
+      RSpec.world.wants_to_quit = false
+
+      # RSpec.clear_examples calls world.reset which clears world.example_groups,
+      # but NOT world.filtered_examples. That hash is keyed by example group
+      # class objects, so old group classes from the previous job remain
+      # referenced as hash keys and cannot be GC'd.
+      RSpec.world.filtered_examples.clear
+
+      # The shared example group registry is also never cleared by world.reset.
+      # When a spec file defines shared_examples/shared_context inside a
+      # describe block, the example group CLASS becomes a key in the registry's
+      # internal hash. Each `load` of a spec file creates new group classes
+      # that get pinned as hash keys, preventing GC of the entire class tree
+      # (including onceler Marshal blobs). We clear per-group entries but
+      # preserve top-level (:main) shared examples from support files, which
+      # are loaded once via `require` and won't be re-defined.
+      RSpec.world.shared_example_group_registry
+           .send(:shared_example_groups)
+           .select! { |k, _| k == :main }
+    end
+
+    # NOTE: RSpec has to load the files before we can split them as individual
+    # examples. In case a file to be splitted fails to be loaded
+    # (e.g. contains a syntax error), we return the files unchanged, thereby
+    # falling back to scheduling them as whole files. Their errors will be
+    # reported in the normal flow when they're eventually picked up by a worker.
+    def files_to_example_ids(files)
+      cmd = "DISABLE_SPRING=1 SUPPRESS_OUTPUT=1 bundle exec rspec --dry-run --format json #{files.join(' ')}"
+      out, err, cmd_result = Open3.capture3(cmd)
+
+      if !cmd_result.success?
+        rspec_output = begin
+          JSON.parse(out)
+        rescue JSON::ParserError
+          out
+        end
+
+        log_event(
+          "Failed to split slow files, falling back to regular scheduling.\n #{err}",
+          "error",
+          rspec_stdout: rspec_output,
+          rspec_stderr: err,
+          cmd_result: cmd_result.inspect
+        )
+
+        pp rspec_output
+
+        return files
+      end
+
+      JSON.parse(out)["examples"].map { |e| e["id"] }
+    end
+
+    def relative_path(job)
+      @cwd ||= Pathname.new(Dir.pwd)
+      "./#{Pathname.new(job).relative_path_from(@cwd)}"
+    end
+
+    def elapsed(since)
+      Process.clock_gettime(Process::CLOCK_MONOTONIC) - since
+    end
+
+    # Prints msg to standard output and emits an event to Sentry, if the
+    # SENTRY_DSN environment variable is set.
+    def log_event(msg, level, additional = {})
+      puts msg
+
+      Sentry.capture_message(msg, level: level, extra: {
+        build: @build_id,
+        worker: @worker_id,
+        queue: queue.inspect,
+        files_or_dirs_to_run: files_or_dirs_to_run,
+        file_split_threshold: file_split_threshold,
+        heartbeat_updated_at: @heartbeat_updated_at,
+        object: inspect,
+        pid: Process.pid
+      }.merge(additional))
+    end
+  end
+end

data/lib/rspecq.rb

@@ -0,0 +1,15 @@
+require "rspec/core"
+require "sentry-ruby"
+
+require_relative "rspecq/formatters/example_count_recorder"
+require_relative "rspecq/formatters/failure_recorder"
+require_relative "rspecq/formatters/job_timing_recorder"
+require_relative "rspecq/formatters/junit_formatter"
+require_relative "rspecq/formatters/worker_heartbeat_recorder"
+
+require_relative "rspecq/configuration"
+require_relative "rspecq/parser"
+require_relative "rspecq/queue"
+require_relative "rspecq/reporter"
+require_relative "rspecq/version"
+require_relative "rspecq/worker"