rspecq 0.5.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2abc0b960b2d28528c0d8a4f9fff7dc0708fd582c7377af021b6c01777350124
-  data.tar.gz: 23d432f09a68ace0932d3c82a112fd0e7058357fa4029c94401a75b84240d5b7
+  metadata.gz: d62ec9f80b28ee9acd347b82b484eeeb0e7a6b21f76e201a19d14477840f91a3
+  data.tar.gz: 8b20f8f635cc3bec96707e29f6547a060baf7131c6e53527e3e1ce8583e5e27b
 SHA512:
-  metadata.gz: 9c499c1ded556a66e9547e71ebfc3f4cdefd6ec984269986b7ae2e1078a86e157ed9805d57afd615f7e9044869124b8f2359095c65febf2d39b3065191e450b7
-  data.tar.gz: 050a8b4eb4983234cbd2d42a24590c9d0914833231c16692f273020c297757c8afca453f09f9ff72a6ec10564809c81356f3ab23c9220644fbcba89f7f0a90a3
+  metadata.gz: 327351debae5f53a3a0be189fc7ff59877030d38d640e3310bb6cd24ec8944e7b38f7761f88e05fa5fa5804d37bbc5fd4889839829f88d96f1d84e1545de6625
+  data.tar.gz: 4ba95049a709afb17feebd22617c7018528fe9f2379170f93913e66e10ddb2723eb01411b228087efe8cfb27a255a7dacce7cddd283246da89c34dbbfe2d5fc0

data/CHANGELOG.md

@@ -1,8 +1,29 @@
 # Changelog
 
+Breaking changes are prefixed with a "[BREAKING]" label.
+
 ## master (unreleased)
 
-Breaking changes are prefixed with a "[BREAKING]" label.
+## 0.7.2 (2021-11-15)
+
+- Add tag option to filter specs (@jorge-wonolo)
+
+## 0.7.1 (2021-04-08)
+
+- New env variable RSPECQ_REPORTER_RERUN_COMMAND_SKIP. When set, the reporter
+  does not include the flaky test's rerun command.
+
+## 0.7.0 (2021-04-01)
+
+- New cli parameter `reproduction`.
+  When passed, primary worker publishes the queue in the same order as passed
+  in the command.
+- Reporter now includes a reproduction command for flaky tests.
+
+## 0.6.0 (2021-03-23)
+
+- New cli parameter `seed`.
+  The seed is passed to the RSpec command.
 
 ## 0.5.0 (2021-02-05)
 

data/README.md

@@ -64,6 +64,7 @@ USAGE:
 OPTIONS:
     -b, --build ID                   A unique identifier for the build. Should be common among workers participating in the same build.
     -w, --worker ID                  An identifier for the worker. Workers participating in the same build should have distinct IDs.
+        --seed SEED                  The RSpec seed. Passing the seed can be helpful in many ways i.e reproduction and testing.
     -r, --redis HOST                 --redis is deprecated. Use --redis-host or --redis-url instead. Redis host to connect to (default: 127.0.0.1).
         --redis-host HOST            Redis host to connect to (default: 127.0.0.1).
         --redis-url URL              Redis URL to connect to (e.g.: redis://127.0.0.1:6379/0).
@@ -75,10 +76,36 @@ OPTIONS:
         --max-requeues N             Retry failed examples up to N times before considering them legit failures (default: 3).
         --queue-wait-timeout N       Time to wait for a queue to be ready before considering it failed (default: 30).
         --fail-fast N                Abort build with a non-zero status code after N failed examples.
+        --reproduction               Enable reproduction mode: Publish files and examples in the exact order given in the command. Incompatible with --timings.
+        --tag TAG                    Run examples with the specified tag, or exclude examples by adding ~ before the tag.  - e.g. ~slow  - TAG is always converted to a symbol.
     -h, --help                       Show this message.
     -v, --version                    Print the version and exit.
 ```
 
+You can set most options using ENV variables:
+
+```shell
+$ RSPECQ_BUILD=123 RSPECQ_WORKER=foo1 rspecq spec/
+```
+
+### Supported ENV variables
+
+| Name | Desc |
+| --- | --- |
+| `RSPECQ_BUILD` | Build ID |
+| `RSPECQ_WORKER` | Worker ID |
+| `RSPECQ_SEED` | RSpec seed |
+| `RSPECQ_REDIS` | Redis HOST |
+| `RSPECQ_UPDATE_TIMINGS` | Timings |
+| `RSPECQ_FILE_SPLIT_THRESHOLD` | File split threshold |
+| `RSPECQ_REPORT` | Report |
+| `RSPECQ_REPORT_TIMEOUT` | Report Timeout |
+| `RSPECQ_MAX_REQUEUES` | Max requests |
+| `RSPECQ_QUEUE_WAIT_TIMEOUT` | Queue wait timeout |
+| `RSPECQ_REDIS_URL` | Redis URL |
+| `RSPECQ_FAIL_FAST` | Fail fast |
+| `RSPECQ_REPORTER_RERUN_COMMAND_SKIP` | Do not report flaky test's rerun command |
+
 ### Sentry integration
 
 RSpecQ can optionally emit build events to a
@@ -90,7 +117,6 @@ This is convenient for monitoring important warnings/errors that may impact
 build times, such as the fact that no previous timings were found and
 therefore job scheduling was effectively random for a particular build.
 
-
 ## How it works
 
 The core design is almost identical to ci-queue so please refer to its
@@ -117,7 +143,7 @@ For example, a single file may need 10 minutes to run while all other
 files finish after 8 minutes. This would cause all but one workers to be
 sitting idle for 2 minutes.
 
-To overcome this issue, RSpecQ can splits files which their execution time is
+To overcome this issue, RSpecQ can split files which their execution time is
 above a certain threshold (set with the `--file-split-threshold` option)
 and instead schedule them as individual examples.
 
@@ -216,6 +242,13 @@ To enable verbose output in the tests:
 $ RSPECQ_DEBUG=1 bundle exec rake
 ```
 
+## Redis
+
+RSpecQ by design doesn't expire its keys from Redis. It is left to the user
+to configure the Redis server to do so; see
+[Using Redis as an LRU cache](https://redis.io/topics/lru-cache) for more info.
+
+You can do this from a configuration file or with `redis-cli`.
 
 ## License
 

data/bin/rspecq

@@ -12,7 +12,9 @@ def env_set?(var)
   ["1", "true"].include?(ENV[var])
 end
 
-opts = {}
+opts = {
+  tags: []
+}
 
 OptionParser.new do |o|
   name = File.basename($PROGRAM_NAME)
@@ -38,9 +40,14 @@ OptionParser.new do |o|
     opts[:worker] = v
   end
 
+  o.on("--seed SEED", "The RSpec seed. Passing the seed can be helpful in " \
+    "many ways i.e reproduction and testing.") do |v|
+    opts[:seed] = v
+  end
+
   o.on("-r", "--redis HOST", "Redis host to connect to " \
        "(default: #{DEFAULT_REDIS_HOST}).") do |v|
-    puts "--redis is deprecated. Use --redis-host or --redis-url instead"
+    puts "DEPRECATION: --redis is deprecated. Use --redis-host or --redis-url instead"
     opts[:redis_host] = v
   end
 
@@ -96,11 +103,24 @@ OptionParser.new do |o|
     opts[:fail_fast] = v
   end
 
+  o.on("--reproduction", "Enable reproduction mode: run rspec on the given files " \
+       "and examples in the exact order they are given. Incompatible with " \
+       "--timings.") do |v|
+    opts[:reproduction] = v
+  end
+
   o.on_tail("-h", "--help", "Show this message.") do
     puts o
     exit
   end
 
+  o.on("--tag TAG", "Run examples with the specified tag, or exclude examples " \
+       "by adding ~ before the tag." \
+       "  - e.g. ~slow" \
+       "  - TAG is always converted to a symbol.") do |tag|
+    opts[:tags] << tag
+  end
+
   o.on_tail("-v", "--version", "Print the version and exit.") do
     puts "#{name} #{RSpecQ::VERSION}"
     exit
@@ -109,6 +129,7 @@ end.parse!
 
 opts[:build] ||= ENV["RSPECQ_BUILD"]
 opts[:worker] ||= ENV["RSPECQ_WORKER"]
+opts[:seed] ||= ENV["RSPECQ_SEED"]
 opts[:redis_host] ||= ENV["RSPECQ_REDIS"] || DEFAULT_REDIS_HOST
 opts[:timings] ||= env_set?("RSPECQ_UPDATE_TIMINGS")
 opts[:file_split_threshold] ||= Integer(ENV["RSPECQ_FILE_SPLIT_THRESHOLD"] || 9_999_999)
@@ -118,6 +139,7 @@ opts[:max_requeues] ||= Integer(ENV["RSPECQ_MAX_REQUEUES"] || DEFAULT_MAX_REQUEU
 opts[:queue_wait_timeout] ||= Integer(ENV["RSPECQ_QUEUE_WAIT_TIMEOUT"] || DEFAULT_QUEUE_WAIT_TIMEOUT)
 opts[:redis_url] ||= ENV["RSPECQ_REDIS_URL"]
 opts[:fail_fast] ||= Integer(ENV["RSPECQ_FAIL_FAST"] || DEFAULT_FAIL_FAST)
+opts[:reproduction] ||= env_set?("RSPECQ_REPRODUCTION")
 
 # rubocop:disable Style/RaiseArgs, Layout/EmptyLineAfterGuardClause
 raise OptionParser::MissingArgument.new(:build) if opts[:build].nil?
@@ -148,11 +170,14 @@ else
     redis_opts: redis_opts
   )
 
-  worker.files_or_dirs_to_run = ARGV[0] if ARGV[0]
+  worker.files_or_dirs_to_run = ARGV if ARGV.any?
   worker.populate_timings = opts[:timings]
   worker.file_split_threshold = opts[:file_split_threshold]
   worker.max_requeues = opts[:max_requeues]
   worker.queue_wait_timeout = opts[:queue_wait_timeout]
   worker.fail_fast = opts[:fail_fast]
+  worker.seed = Integer(opts[:seed]) if opts[:seed]
+  worker.reproduction = opts[:reproduction]
+  worker.tags = opts[:tags]
   worker.work
 end

data/lib/rspecq/formatters/failure_recorder.rb

@@ -6,12 +6,13 @@ module RSpecQ
     # Also persists non-example error information (e.g. a syntax error that
     # in a spec file).
     class FailureRecorder
-      def initialize(queue, job, max_requeues)
+      def initialize(queue, job, max_requeues, worker_id)
         @queue = queue
         @job = job
         @colorizer = RSpec::Core::Formatters::ConsoleCodes
         @non_example_error_recorded = false
         @max_requeues = max_requeues
+        @worker_id = worker_id
       end
 
       # Here we're notified about errors occuring outside of examples.
@@ -30,7 +31,7 @@ module RSpecQ
       def example_failed(notification)
         example = notification.example
 
-        if @queue.requeue_job(example.id, @max_requeues)
+        if @queue.requeue_job(example, @max_requeues, @worker_id)
           # HACK: try to avoid picking the job we just requeued; we want it
           # to be picked up by a different worker
           sleep 0.5
@@ -44,7 +45,7 @@ module RSpecQ
         msg = presenter.fully_formatted(nil, @colorizer)
         msg << "\n"
         msg << @colorizer.wrap(
-          "bin/rspec #{example.location_rerun_argument}",
+          "bin/rspec --seed #{RSpec.configuration.seed} #{example.location_rerun_argument}",
           RSpec.configuration.failure_color
         )
 

data/lib/rspecq/queue.rb

@@ -51,8 +51,12 @@ module RSpecQ
     REQUEUE_JOB = <<~LUA.freeze
       local key_queue_unprocessed = KEYS[1]
       local key_requeues = KEYS[2]
+      local key_requeued_job_original_worker = KEYS[3]
+      local key_job_location = KEYS[4]
       local job = ARGV[1]
       local max_requeues = ARGV[2]
+      local original_worker = ARGV[3]
+      local location = ARGV[4]
 
       local requeued_times = redis.call('hget', key_requeues, job)
       if requeued_times and requeued_times >= max_requeues then
@@ -60,7 +64,9 @@ module RSpecQ
       end
 
       redis.call('lpush', key_queue_unprocessed, job)
+      redis.call('hset', key_requeued_job_original_worker, job, original_worker)
       redis.call('hincrby', key_requeues, job, 1)
+      redis.call('hset', key_job_location, job, location)
 
       return true
     LUA
@@ -117,6 +123,7 @@ module RSpecQ
       @redis.multi do
         @redis.hdel(key_queue_running, @worker_id)
         @redis.sadd(key_queue_processed, job)
+        @redis.rpush(key("queue", "jobs_per_worker", @worker_id), job)
       end
     end
 
@@ -125,16 +132,41 @@ module RSpecQ
     #
     # Returns nil if the job hit the requeue limit and therefore was not
     # requeued and should be considered a failure.
-    def requeue_job(job, max_requeues)
+    def requeue_job(example, max_requeues, original_worker_id)
       return false if max_requeues.zero?
 
+      job = example.id
+      location = example.location_rerun_argument
+
       @redis.eval(
         REQUEUE_JOB,
-        keys: [key_queue_unprocessed, key_requeues],
-        argv: [job, max_requeues]
+        keys: [key_queue_unprocessed, key_requeues, key("requeued_job_original_worker"), key("job_location")],
+        argv: [job, max_requeues, original_worker_id, location]
       )
     end
 
+    def save_worker_seed(worker, seed)
+      @redis.hset(key("worker_seed"), worker, seed)
+    end
+
+    def job_location(job)
+      @redis.hget(key("job_location"), job)
+    end
+
+    def failed_job_worker(job)
+      redis.hget(key("requeued_job_original_worker"), job)
+    end
+
+    def job_rerun_command(job)
+      worker = failed_job_worker(job)
+      jobs = redis.lrange(key("queue", "jobs_per_worker", worker), 0, -1)
+      seed = redis.hget(key("worker_seed"), worker)
+
+      "DISABLE_SPRING=1 DISABLE_BOOTSNAP=1 bin/rspecq --build 1 " \
+        "--worker foo --seed #{seed} --max-requeues 0 --fail-fast 1 " \
+        "--reproduction #{jobs.join(' ')}"
+    end
+
     def record_example_failure(example_id, message)
       @redis.hset(key_failures, example_id, message)
     end

data/lib/rspecq/reporter.rb

@@ -107,7 +107,16 @@ module RSpecQ
       if !flaky_jobs.empty?
         summary << "\n\n"
         summary << "Flaky jobs detected (count=#{flaky_jobs.count}):\n"
-        flaky_jobs.each { |j| summary << "  #{j}\n" }
+        flaky_jobs.each do |j|
+          summary << RSpec::Core::Formatters::ConsoleCodes.wrap(
+            "#{@queue.job_location(j)} @ #{@queue.failed_job_worker(j)}\n",
+            RSpec.configuration.pending_color
+          )
+
+          next if ENV["RSPECQ_REPORTER_RERUN_COMMAND_SKIP"]
+
+          summary << "#{@queue.job_rerun_command(j)}\n\n\n"
+        end
       end
 
       summary
@@ -125,16 +134,15 @@ module RSpecQ
       return if jobs.empty?
 
       jobs.each do |job|
-        filename = job.sub(/\[.+\]/, "")
+        filename = job.sub(/\[.+\]/, "")[%r{spec/.+}].split(":")[0]
 
         extra = {
           build: @build_id,
           build_timeout: @timeout,
-          queue: @queue.inspect,
-          object: inspect,
-          pid: Process.pid,
-          job_path: job,
-          build_duration: build_duration
+          build_duration: build_duration,
+          location: @queue.job_location(job),
+          rerun_command: @queue.job_rerun_command(job),
+          worker: @queue.failed_job_worker(job)
         }
 
         tags = {

data/lib/rspecq/version.rb

@@ -1,3 +1,3 @@
 module RSpecQ
-  VERSION = "0.5.0".freeze
+  VERSION = "0.7.2".freeze
 end

data/lib/rspecq/worker.rb

@@ -51,6 +51,16 @@ module RSpecQ
     # Defaults to 30
     attr_accessor :queue_wait_timeout
 
+    # The RSpec seed
+    attr_accessor :seed
+
+    # Rspec tags
+    attr_accessor :tags
+
+    # Reproduction flag. If true, worker will publish files in the exact order
+    # given in the command.
+    attr_accessor :reproduction
+
     attr_reader :queue
 
     def initialize(build_id:, worker_id:, redis_opts:)
@@ -64,6 +74,9 @@ module RSpecQ
       @heartbeat_updated_at = nil
       @max_requeues = 3
       @queue_wait_timeout = 30
+      @seed = srand && srand % 0xFFFF
+      @tags = []
+      @reproduction = false
 
       RSpec::Core::Formatters.register(Formatters::JobTimingRecorder, :dump_summary)
       RSpec::Core::Formatters.register(Formatters::ExampleCountRecorder, :dump_summary)
@@ -76,6 +89,7 @@ module RSpecQ
 
       try_publish_queue!(queue)
       queue.wait_until_published(queue_wait_timeout)
+      queue.save_worker_seed(@worker_id, seed)
 
       loop do
         # we have to bootstrap this so that it can be used in the first call
@@ -103,17 +117,19 @@ module RSpecQ
 
         # reconfigure rspec
         RSpec.configuration.detail_color = :magenta
-        RSpec.configuration.seed = srand && srand % 0xFFFF
+        RSpec.configuration.seed = seed
         RSpec.configuration.backtrace_formatter.filter_gem("rspecq")
-        RSpec.configuration.add_formatter(Formatters::FailureRecorder.new(queue, job, max_requeues))
+        RSpec.configuration.add_formatter(Formatters::FailureRecorder.new(queue, job, max_requeues, @worker_id))
         RSpec.configuration.add_formatter(Formatters::ExampleCountRecorder.new(queue))
         RSpec.configuration.add_formatter(Formatters::WorkerHeartbeatRecorder.new(self))
 
         if populate_timings
           RSpec.configuration.add_formatter(Formatters::JobTimingRecorder.new(queue, job))
         end
-
-        opts = RSpec::Core::ConfigurationOptions.new(["--format", "progress", job])
+        
+        options = ["--format", "progress", job]
+        tags.each { |tag| options.push(*["--tag", tag]) }
+        opts = RSpec::Core::ConfigurationOptions.new(options)
         _result = RSpec::Core::Runner.new(opts).run($stderr, $stdout)
 
         queue.acknowledge_job(job)
@@ -131,6 +147,14 @@ module RSpecQ
     def try_publish_queue!(queue)
       return if !queue.become_master
 
+      if reproduction
+        q_size = queue.publish(files_or_dirs_to_run, fail_fast)
+        log_event(
+          "Reproduction mode. Published queue as given (size=#{q_size})",
+          "info"
+        )
+        return
+      end
       RSpec.configuration.files_or_directories_to_run = files_or_dirs_to_run
       files_to_run = RSpec.configuration.files_to_run.map { |j| relative_path(j) }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspecq
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Agis Anastasopoulos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-05 00:00:00.000000000 Z
+date: 2021-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core