rspecq 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86b8792f2ff107cbfb1e67b881724c535df7730804feec7e4813f4690795fac1
-  data.tar.gz: bbd5b78cf7919b653446b00c0748c0ea0d9fca32f69e40e7dd439b3396042dec
+  metadata.gz: 4337137edc0f3ee2c22d4f9cd2547ab31d4ddc0b043b49848c7b513ae98094a9
+  data.tar.gz: 6de02e1ce31cb7dff6af9d4862f1dceebfbfbbd10063329c9d3d058d32bf8f06
 SHA512:
-  metadata.gz: 29ee44880ef2d654cd8faeeeb7bcdee2f7ff2b468a22a51a3831d6a175a31246e0cd2206ce012eaa3c1a2ad3aad319360baa10f10d3b84745e7ee9e58fdf3b7b
-  data.tar.gz: d8cd7a9515a25a9b8679acffa99f3b8f19764ff7e23079921677aa40b1941cb6fbce05dbad813f308d68d460f7c06b97a453de2622ee8060161033f65c889189
+  metadata.gz: beb1b013a1d78e8c18b473ef33d286224a92ef46ae3944f99296cc3d2d13193309b0efb020307fa3e9139b7fd88eb9a9111ad526026762ccf0b17ba892b3ddb4
+  data.tar.gz: 178481fe5b707cefa25426ec6b2ead691dd69494a6b9869eaf8e438b78e100c5dd8c0c9caab2dd4b792c18e5c931e4e7231cc971b77fa048c14420f33978fe2a

data/CHANGELOG.md

@@ -4,6 +4,13 @@ Breaking changes are prefixed with a "[BREAKING]" label.
 
 ## master (unreleased)
 
+## 0.7.0 (2021-04-1)
+
+- New cli parameter `reproduction`.
+  When passed, primary worker publishes the queue in the same order as passed
+  in the command.
+- Reporter now includes a reproduction command for flaky tests.
+
 ## 0.6.0 (2021-03-23)
 
 - New cli parameter `seed`.

data/README.md

@@ -64,6 +64,7 @@ USAGE:
 OPTIONS:
     -b, --build ID                   A unique identifier for the build. Should be common among workers participating in the same build.
     -w, --worker ID                  An identifier for the worker. Workers participating in the same build should have distinct IDs.
+        --seed SEED                  The RSpec seed. Passing the seed can be helpful in many ways i.e reproduction and testing.
     -r, --redis HOST                 --redis is deprecated. Use --redis-host or --redis-url instead. Redis host to connect to (default: 127.0.0.1).
         --redis-host HOST            Redis host to connect to (default: 127.0.0.1).
         --redis-url URL              Redis URL to connect to (e.g.: redis://127.0.0.1:6379/0).
@@ -75,6 +76,7 @@ OPTIONS:
         --max-requeues N             Retry failed examples up to N times before considering them legit failures (default: 3).
         --queue-wait-timeout N       Time to wait for a queue to be ready before considering it failed (default: 30).
         --fail-fast N                Abort build with a non-zero status code after N failed examples.
+        --reproduction               Enable reproduction mode: Publish files and examples in the exact order given in the command. Incompatible with --timings.
     -h, --help                       Show this message.
     -v, --version                    Print the version and exit.
 ```

data/bin/rspecq

@@ -101,6 +101,12 @@ OptionParser.new do |o|
     opts[:fail_fast] = v
   end
 
+  o.on("--reproduction", "Enable reproduction mode: run rspec on the given files " \
+       "and examples in the exact order they are given. Incompatible with " \
+       "--timings.") do |v|
+    opts[:reproduction] = v
+  end
+
   o.on_tail("-h", "--help", "Show this message.") do
     puts o
     exit
@@ -124,6 +130,7 @@ opts[:max_requeues] ||= Integer(ENV["RSPECQ_MAX_REQUEUES"] || DEFAULT_MAX_REQUEU
 opts[:queue_wait_timeout] ||= Integer(ENV["RSPECQ_QUEUE_WAIT_TIMEOUT"] || DEFAULT_QUEUE_WAIT_TIMEOUT)
 opts[:redis_url] ||= ENV["RSPECQ_REDIS_URL"]
 opts[:fail_fast] ||= Integer(ENV["RSPECQ_FAIL_FAST"] || DEFAULT_FAIL_FAST)
+opts[:reproduction] ||= env_set?("RSPECQ_REPRODUCTION")
 
 # rubocop:disable Style/RaiseArgs, Layout/EmptyLineAfterGuardClause
 raise OptionParser::MissingArgument.new(:build) if opts[:build].nil?
@@ -154,12 +161,13 @@ else
     redis_opts: redis_opts
   )
 
-  worker.files_or_dirs_to_run = ARGV[0] if ARGV[0]
+  worker.files_or_dirs_to_run = ARGV if ARGV.any?
   worker.populate_timings = opts[:timings]
   worker.file_split_threshold = opts[:file_split_threshold]
   worker.max_requeues = opts[:max_requeues]
   worker.queue_wait_timeout = opts[:queue_wait_timeout]
   worker.fail_fast = opts[:fail_fast]
   worker.seed = Integer(opts[:seed]) if opts[:seed]
+  worker.reproduction = opts[:reproduction]
   worker.work
 end

data/lib/rspecq/formatters/failure_recorder.rb

@@ -6,12 +6,13 @@ module RSpecQ
     # Also persists non-example error information (e.g. a syntax error that
     # in a spec file).
     class FailureRecorder
-      def initialize(queue, job, max_requeues)
+      def initialize(queue, job, max_requeues, worker_id)
         @queue = queue
         @job = job
         @colorizer = RSpec::Core::Formatters::ConsoleCodes
         @non_example_error_recorded = false
         @max_requeues = max_requeues
+        @worker_id = worker_id
       end
 
       # Here we're notified about errors occuring outside of examples.
@@ -30,14 +31,7 @@ module RSpecQ
       def example_failed(notification)
         example = notification.example
 
-        rerun_cmd = "bin/rspec --seed #{RSpec.configuration.seed} #{example.location_rerun_argument}"
-
-        if @queue.requeue_job(example.id, @max_requeues)
-
-          # Save the rerun command for later. It will be used if this is
-          # a flaky test for more user-friendly reporting.
-          @queue.save_rerun_command(example.id, rerun_cmd)
-
+        if @queue.requeue_job(example, @max_requeues, @worker_id)
           # HACK: try to avoid picking the job we just requeued; we want it
           # to be picked up by a different worker
           sleep 0.5
@@ -51,7 +45,7 @@ module RSpecQ
         msg = presenter.fully_formatted(nil, @colorizer)
         msg << "\n"
         msg << @colorizer.wrap(
-          rerun_cmd,
+          "bin/rspec --seed #{RSpec.configuration.seed} #{example.location_rerun_argument}",
           RSpec.configuration.failure_color
         )
 

data/lib/rspecq/queue.rb

@@ -51,8 +51,12 @@ module RSpecQ
     REQUEUE_JOB = <<~LUA.freeze
       local key_queue_unprocessed = KEYS[1]
       local key_requeues = KEYS[2]
+      local key_requeued_job_original_worker = KEYS[3]
+      local key_job_location = KEYS[4]
       local job = ARGV[1]
       local max_requeues = ARGV[2]
+      local original_worker = ARGV[3]
+      local location = ARGV[4]
 
       local requeued_times = redis.call('hget', key_requeues, job)
       if requeued_times and requeued_times >= max_requeues then
@@ -60,7 +64,9 @@ module RSpecQ
       end
 
       redis.call('lpush', key_queue_unprocessed, job)
+      redis.call('hset', key_requeued_job_original_worker, job, original_worker)
       redis.call('hincrby', key_requeues, job, 1)
+      redis.call('hset', key_job_location, job, location)
 
       return true
     LUA
@@ -117,6 +123,7 @@ module RSpecQ
       @redis.multi do
         @redis.hdel(key_queue_running, @worker_id)
         @redis.sadd(key_queue_processed, job)
+        @redis.rpush(key("queue", "jobs_per_worker", @worker_id), job)
       end
     end
 
@@ -125,22 +132,39 @@ module RSpecQ
     #
     # Returns nil if the job hit the requeue limit and therefore was not
     # requeued and should be considered a failure.
-    def requeue_job(job, max_requeues)
+    def requeue_job(example, max_requeues, original_worker_id)
       return false if max_requeues.zero?
 
+      job = example.id
+      location = example.location_rerun_argument
+
       @redis.eval(
         REQUEUE_JOB,
-        keys: [key_queue_unprocessed, key_requeues],
-        argv: [job, max_requeues]
+        keys: [key_queue_unprocessed, key_requeues, key("requeued_job_original_worker"), key("job_location")],
+        argv: [job, max_requeues, original_worker_id, location]
       )
     end
 
-    def save_rerun_command(job, cmd)
-      @redis.hset(key("job_metadata"), job, cmd)
+    def save_worker_seed(worker, seed)
+      @redis.hset(key("worker_seed"), worker, seed)
+    end
+
+    def job_location(job)
+      @redis.hget(key("job_location"), job)
     end
 
-    def rerun_command(job)
-      @redis.hget(key("job_metadata"), job)
+    def failed_job_worker(job)
+      redis.hget(key("requeued_job_original_worker"), job)
+    end
+
+    def job_rerun_command(job)
+      worker = failed_job_worker(job)
+      jobs = redis.lrange(key("queue", "jobs_per_worker", worker), 0, -1)
+      seed = redis.hget(key("worker_seed"), worker)
+
+      "DISABLE_SPRING=1 DISABLE_BOOTSNAP=1 bin/rspecq --build 1 " \
+        "--worker foo --seed #{seed} --max-requeues 0 --fail-fast 1 " \
+        "--reproduction #{jobs.join(' ')}"
     end
 
     def record_example_failure(example_id, message)

data/lib/rspecq/reporter.rb

@@ -56,7 +56,7 @@ module RSpecQ
 
       @queue.record_build_time(tests_duration)
 
-      flaky_jobs = @queue.flaky_jobs.map { |job| @queue.rerun_command(job) }
+      flaky_jobs = @queue.flaky_jobs
 
       puts summary(@queue.example_failures, @queue.non_example_errors,
         flaky_jobs, humanize_duration(tests_duration))
@@ -109,9 +109,11 @@ module RSpecQ
         summary << "Flaky jobs detected (count=#{flaky_jobs.count}):\n"
         flaky_jobs.each do |j|
           summary << RSpec::Core::Formatters::ConsoleCodes.wrap(
-            "#{j}\n",
+            "#{@queue.job_location(j)} @ #{@queue.failed_job_worker(j)}\n",
             RSpec.configuration.pending_color
           )
+
+          summary << "#{@queue.job_rerun_command(j)}\n\n\n"
         end
       end
 
@@ -130,16 +132,15 @@ module RSpecQ
       return if jobs.empty?
 
       jobs.each do |job|
-        filename = job.sub(/\[.+\]/, "")
+        filename = job.sub(/\[.+\]/, "")[%r{spec/.+}].split(":")[0]
 
         extra = {
           build: @build_id,
           build_timeout: @timeout,
-          queue: @queue.inspect,
-          object: inspect,
-          pid: Process.pid,
-          job_path: job,
-          build_duration: build_duration
+          build_duration: build_duration,
+          location: @queue.job_location(job),
+          rerun_command: @queue.job_rerun_command(job),
+          worker: @queue.failed_job_worker(job)
         }
 
         tags = {

data/lib/rspecq/version.rb

@@ -1,3 +1,3 @@
 module RSpecQ
-  VERSION = "0.6.0".freeze
+  VERSION = "0.7.0".freeze
 end

data/lib/rspecq/worker.rb

@@ -54,6 +54,10 @@ module RSpecQ
     # The RSpec seed
     attr_accessor :seed
 
+    # Reproduction flag. If true, worker will publish files in the exact order
+    # given in the command.
+    attr_accessor :reproduction
+
     attr_reader :queue
 
     def initialize(build_id:, worker_id:, redis_opts:)
@@ -68,6 +72,7 @@ module RSpecQ
       @max_requeues = 3
       @queue_wait_timeout = 30
       @seed = srand && srand % 0xFFFF
+      @reproduction = false
 
       RSpec::Core::Formatters.register(Formatters::JobTimingRecorder, :dump_summary)
       RSpec::Core::Formatters.register(Formatters::ExampleCountRecorder, :dump_summary)
@@ -80,6 +85,7 @@ module RSpecQ
 
       try_publish_queue!(queue)
       queue.wait_until_published(queue_wait_timeout)
+      queue.save_worker_seed(@worker_id, seed)
 
       loop do
         # we have to bootstrap this so that it can be used in the first call
@@ -109,7 +115,7 @@ module RSpecQ
         RSpec.configuration.detail_color = :magenta
         RSpec.configuration.seed = seed
         RSpec.configuration.backtrace_formatter.filter_gem("rspecq")
-        RSpec.configuration.add_formatter(Formatters::FailureRecorder.new(queue, job, max_requeues))
+        RSpec.configuration.add_formatter(Formatters::FailureRecorder.new(queue, job, max_requeues, @worker_id))
         RSpec.configuration.add_formatter(Formatters::ExampleCountRecorder.new(queue))
         RSpec.configuration.add_formatter(Formatters::WorkerHeartbeatRecorder.new(self))
 
@@ -135,6 +141,15 @@ module RSpecQ
     def try_publish_queue!(queue)
       return if !queue.become_master
 
+      if reproduction
+        q_size = queue.publish(files_or_dirs_to_run, fail_fast)
+        log_event(
+          "Reproduction mode. Published queue as given (size=#{q_size})",
+          "info"
+        )
+        return
+      end
+
       RSpec.configuration.files_or_directories_to_run = files_or_dirs_to_run
       files_to_run = RSpec.configuration.files_to_run.map { |j| relative_path(j) }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspecq
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Agis Anastasopoulos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-23 00:00:00.000000000 Z
+date: 2021-04-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core