rspecq 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89dbfa98d1eaceb06c39d41ab85e7fa6923d0c87e9a15b9cbfaf7399ff2aaff3
-  data.tar.gz: b7cd028440e6eb03401dc623c7ee0fc0fe74f6ffa12a25ecc23d0cf54e6acd1e
+  metadata.gz: ae9ce385f55d300aa1c6d50b75ffe5bb91b28783e834009dc308d3e52856f786
+  data.tar.gz: 13cf6b430cb88a14040442a94f0df5d2831f263ae1205a6b0c6e228c5bc3aefe
 SHA512:
-  metadata.gz: a43f0630e8a02a001132f45c9f68cacf7edae8e90487112e640eb611e7d1345f68ad0ab163ae01c91cb38ebc879e98cda9b54044c0ee676293c7aa3bf7c17942
-  data.tar.gz: bf98027dc02ac56d02cc258700f5efa766c40d4d69c55c529d311083965d1fe4f76423b05fddb35a37496bb6eb3c11a8460a8e76f94897c4bb38a744b2fb40df
+  metadata.gz: a98eadc353fbc02963228503b92d88cdc04414c037d5728ebad471764b202967a5b535e34a9533abb8879ccfe20b88ff2561f8e122433b0f6deb1333ee9c22cf
+  data.tar.gz: 58af317b915a83bb8f0458dd25ab7a2b1aa31ee3835e3ad05082a6478f68d74681c0d040d9455b11135e8952bee2aadde70a344d1f25301af0a604a4e996b746

data/CHANGELOG.md

@@ -4,6 +4,14 @@ Breaking changes are prefixed with a "[BREAKING]" label.
 
 ## master (unreleased)
 
+## 0.4.0 (2020-10-07)
+
+### Added
+
+- Builds can be configured to terminate after a specified number of failures,
+  using the `--fail-fast` option.
+
+
 ## 0.3.0 (2020-10-05)
 
 ### Added

data/README.md

@@ -28,6 +28,7 @@ and [ci-queue](https://github.com/Shopify/ci-queue).
 - Sentry integration for monitoring build-level events. See [*Sentry integration*](#sentry-integration).
 - [PLANNED] StatsD integration for various build-level metrics and insights.
   See [#2](https://github.com/skroutz/rspecq/issues/2).
+- Automatic termination of builds after a certain amount of failures. See [*Fail-fast*](#fail-fast).
 
 ## Usage
 
@@ -73,6 +74,7 @@ OPTIONS:
                                      Exits with a non-zero status code if there were any failures.
         --report-timeout N           Fail if build is not finished after N seconds. Only applicable if --report is enabled (default: 3600).
         --max-requeues N             Retry failed examples up to N times before considering them legit failures (default: 3).
+        --fail-fast N                Abort build with a non-zero status code after N failed examples.
     -h, --help                       Show this message.
     -v, --version                    Print the version and exit.
 ```
@@ -133,6 +135,16 @@ final report.
 Flaky tests are also detected and printed as such in the final report. They are
 also emitted to Sentry (see [Sentry integration](#sentry-integration)).
 
+### Fail-fast
+
+In order to prevent large suites running for a long time with a lot of
+failures, a threshold can be set to control the number of failed examples that
+will render the build unsuccessful. This is in par with RSpec's
+[--fail-fast](https://relishapp.com/rspec/rspec-core/docs/command-line/fail-fast-option).
+
+This feature is disabled by default, and can be controlled via the
+`--fail-fast` command line option.
+
 ### Worker failures
 
 It's not uncommon for CI processes to encounter unrecoverable failures for

data/bin/rspecq

@@ -5,6 +5,7 @@ require "rspecq"
 DEFAULT_REDIS_HOST = "127.0.0.1"
 DEFAULT_REPORT_TIMEOUT = 3600 # 1 hour
 DEFAULT_MAX_REQUEUES = 3
+DEFAULT_FAIL_FAST = 0
 
 def env_set?(var)
   ["1", "true"].include?(ENV[var])
@@ -83,6 +84,11 @@ OptionParser.new do |o|
     opts[:max_requeues] = v
   end
 
+  o.on("--fail-fast N", Integer, "Abort build with a non-zero status code " \
+       "after N failed examples." ) do |v|
+    opts[:fail_fast] = v
+  end
+
   o.on_tail("-h", "--help", "Show this message.") do
     puts o
     exit
@@ -103,6 +109,7 @@ opts[:report] ||= env_set?("RSPECQ_REPORT")
 opts[:report_timeout] ||= Integer(ENV["RSPECQ_REPORT_TIMEOUT"] || DEFAULT_REPORT_TIMEOUT)
 opts[:max_requeues] ||= Integer(ENV["RSPECQ_MAX_REQUEUES"] || DEFAULT_MAX_REQUEUES)
 opts[:redis_url] ||= ENV["RSPECQ_REDIS_URL"]
+opts[:fail_fast] ||= Integer(ENV["RSPECQ_FAIL_FAST"] || DEFAULT_FAIL_FAST)
 
 raise OptionParser::MissingArgument.new(:build) if opts[:build].nil?
 raise OptionParser::MissingArgument.new(:worker) if !opts[:report] && opts[:worker].nil?
@@ -134,5 +141,6 @@ else
   worker.populate_timings = opts[:timings]
   worker.file_split_threshold = opts[:file_split_threshold]
   worker.max_requeues = opts[:max_requeues]
+  worker.fail_fast = opts[:fail_fast]
   worker.work
 end

data/lib/rspecq/queue.rb

@@ -77,8 +77,9 @@ module RSpecQ
     end
 
     # NOTE: jobs will be processed from head to tail (lpop)
-    def publish(jobs)
+    def publish(jobs, fail_fast = 0)
       @redis.multi do
+        @redis.hset(key_queue_config, 'fail_fast', fail_fast)
         @redis.rpush(key_queue_unprocessed, jobs)
         @redis.set(key_queue_status, STATUS_READY)
       end.first
@@ -232,7 +233,9 @@ module RSpecQ
     # after being retried). Must be called after the build is complete,
     # otherwise an exception will be raised.
     def flaky_jobs
-      raise "Queue is not yet exhausted" if !exhausted?
+      if !exhausted? && !build_failed_fast?
+        raise "Queue is not yet exhausted"
+      end
 
       requeued = @redis.hkeys(key_requeues)
 
@@ -241,11 +244,38 @@ module RSpecQ
       requeued - @redis.hkeys(key_failures)
     end
 
+    # Returns the number of failures that will trigger the build to fail-fast.
+    # Returns 0 if this feature is disabled and nil if the Queue is not yet
+    # published
+    def fail_fast
+      return nil unless published?
+
+      @fail_fast ||= Integer(@redis.hget(key_queue_config, 'fail_fast'))
+    end
+
+    # Returns true if the number of failed tests, has surpassed the threshold
+    # to render the run unsuccessful and the build should be terminated.
+    def build_failed_fast?
+      if fail_fast.nil? || fail_fast.zero?
+        return false
+      end
+
+      @redis.multi do
+        @redis.hlen(key_failures)
+        @redis.hlen(key_errors)
+      end.inject(:+) >= fail_fast
+    end
+
     # redis: STRING [STATUS_INITIALIZING, STATUS_READY]
     def key_queue_status
       key("queue", "status")
     end
 
+    # redis:  HASH<config_key => config_value>
+    def key_queue_config
+      key("queue", "config")
+    end
+
     # redis: LIST<job>
     def key_queue_unprocessed
       key("queue", "unprocessed")

data/lib/rspecq/reporter.rb

@@ -41,7 +41,7 @@ module RSpecQ
             puts failure_formatted(rspec_output)
           end
 
-          if !@queue.exhausted?
+          unless @queue.exhausted? || @queue.build_failed_fast?
             sleep 1
             next
           end
@@ -83,6 +83,13 @@ module RSpecQ
       end
 
       summary = ""
+      if @queue.build_failed_fast?
+        summary << "\n\n"
+        summary << "The limit of #{@queue.fail_fast} failures has been reached\n"
+        summary << "Aborting..."
+        summary << "\n"
+      end
+
       summary << failed_examples_section if !failures.empty?
 
       errors.each { |_job, msg| summary << msg }

data/lib/rspecq/version.rb

@@ -1,3 +1,3 @@
 module RSpecQ
-  VERSION = "0.3.0".freeze
+  VERSION = "0.4.0".freeze
 end

data/lib/rspecq/worker.rb

@@ -40,12 +40,19 @@ module RSpecQ
     # Defaults to 3
     attr_accessor :max_requeues
 
+    # Stop the execution after N failed tests. Do not stop at any point when
+    # set to 0.
+    #
+    # Defaults to 0
+    attr_accessor :fail_fast
+
     attr_reader :queue
 
     def initialize(build_id:, worker_id:, redis_opts:)
       @build_id = build_id
       @worker_id = worker_id
       @queue = Queue.new(build_id, worker_id, redis_opts)
+      @fail_fast = 0
       @files_or_dirs_to_run = "spec"
       @populate_timings = false
       @file_split_threshold = 999999
@@ -69,6 +76,8 @@ module RSpecQ
         # to `requeue_lost_job` inside the work loop
         update_heartbeat
 
+        return if queue.build_failed_fast?
+
         lost = queue.requeue_lost_job
         puts "Requeued lost job: #{lost}" if lost
 
@@ -121,7 +130,7 @@ module RSpecQ
 
       timings = queue.timings
       if timings.empty?
-        q_size = queue.publish(files_to_run.shuffle)
+        q_size = queue.publish(files_to_run.shuffle, fail_fast)
         log_event(
           "No timings found! Published queue in random order (size=#{q_size})",
           "warning"
@@ -160,7 +169,7 @@ module RSpecQ
       # sort jobs based on their timings (slowest to be processed first)
       jobs = jobs.sort_by { |_j, t| -t }.map(&:first)
 
-      puts "Published queue (size=#{queue.publish(jobs)})"
+      puts "Published queue (size=#{queue.publish(jobs, fail_fast)})"
     end
 
     private

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspecq
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Agis Anastasopoulos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2020-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-core