specwrk 0.13.1 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3028afb58991e7fc5837333b22d842ba261c6fa5985809db231b21fb40fa904
-  data.tar.gz: b3471eafa04b678db1b14cf0684296677dbe995fb2f3a5f920659479110e291f
+  metadata.gz: 7890636149b3658e663cfc5957522e9147f0ede2a6ff4e28c27b6966c1b45b40
+  data.tar.gz: a76c0bd696a11d4b1703dabd156466611678bcec30b35fd6994ced48444e9dcf
 SHA512:
-  metadata.gz: 5ed970474e4a193778c33c113f374ffe8146dca152fb14e991f4a2b1970d1b71fee7050b78029ecd7fd3594b16dbd594d94cacd00e29e98ba7eeefc77c8bed90
-  data.tar.gz: 6a22138e628cd93aa6db4ceaf06d04b702d540765c53c41a0cefe04192af218394a0c5df6aec0200c5e3fcaadae8d521d3b9839290422b562819b6c78701b005
+  metadata.gz: 9ad2c96e0111c5b523836f22a3eda9133f7ef838b5239c8355118a299c9ac6ea523bdb524955bf7b66caffea1397d5470cfd3554e877247268fce332ba34ac43
+  data.tar.gz: dec3475151008f2e4452ed4f5e96c413d63d61c4b2ebe38d315a3d795bed58247341c53c60b8dd7f15385860a1d381d77f60cdba61ce5738c24fb0f27a50b72f

data/.circleci/config.yml

@@ -73,7 +73,8 @@ jobs:
       ## SPECWRK STEP ##
       - run:
           name: Run tests via specwrk start
-          command: bundle exec specwrk start --count 2 spec/
+          # This run will output a single failure when healthy, but because of retries the exit code will be zero
+          command: bundle exec specwrk start --count 2 --max-retries 1 spec/
       ## /SPECWRK STEP ##
         
       ## SPECWRK STEP ##

data/README.md

@@ -1,5 +1,5 @@
 # Specwrk
-Run your [RSpec](https://github.com/rspec/rspec) examples across many processors and many nodes for a single build. Or just many processes on a single node. Speeds up your *slow* (minutes/hours not seconds) test suite by running multiple examples in parallel.
+Run your [RSpec](https://github.com/rspec/rspec) examples across many processors and many nodes for a single build. Or just many processes on a single node. Speeds up your *slow* (minutes/hours not seconds) test suite by running multiple examples in parallel. Optionally, retry failed examples up-to N-times to avoid breaking deployments.
 
 One CLI command to:
 
@@ -60,6 +60,7 @@ Options:
   --store-uri=VALUE                 # Directory where server state is stored. Required for multi-node or multi-process servers.
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
   --[no-]verbose                    # Run in verbose mode, default: false
+  --max-retries=VALUE               # Number of times an example will be re-run should it fail, default: 0
   --help, -h                        # Print this help
 ```
 
@@ -111,6 +112,7 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN, default: "main"
   --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT, default: "5"
+  --max-retries=VALUE               # Number of times an example will be re-run should it fail, default: 0
   --help, -h                        # Print this help
 ```
 

data/lib/specwrk/cli.rb

@@ -49,16 +49,31 @@ module Specwrk
       end
 
       def start_workers
+        @final_outputs = []
         @worker_pids = worker_count.times.map do |i|
+          reader, writer = IO.pipe
+          @final_outputs << reader
+
           Process.fork do
             ENV["TEST_ENV_NUMBER"] = ENV["SPECWRK_FORKED"] = (i + 1).to_s
             ENV["SPECWRK_ID"] = ENV["SPECWRK_ID"] + "-#{i + 1}"
 
+            $final_output = writer # standard:disable Style/GlobalVars
+            $final_output.sync = true # standard:disable Style/GlobalVars
+            reader.close
+
             require "specwrk/worker"
 
             status = Specwrk::Worker.run!
+            $final_output.close # standard:disable Style/GlobalVars
             exit(status)
-          end
+          end.tap { writer.close }
+        end
+      end
+
+      def drain_outputs
+        @final_outputs.each do |reader|
+          reader.each_line { |line| $stdout.print line }
         end
       end
 
@@ -104,10 +119,10 @@ module Specwrk
       include Clientable
 
       desc "Seed the server with a list of specs for the run"
-
+      option :max_retries, default: 0, desc: "Number of times an example will be re-run should it fail"
       argument :dir, required: false, default: "spec", desc: "Relative spec directory to run against"
 
-      def call(dir:, **args)
+      def call(max_retries:, dir:, **args)
         self.class.setup(**args)
 
         require "specwrk/list_examples"
@@ -117,7 +132,7 @@ module Specwrk
         examples = ListExamples.new(dir).examples
 
         Client.wait_for_server!
-        Client.new.seed(examples)
+        Client.new.seed(examples, max_retries: max_retries)
         file_count = examples.group_by { |e| e[:file_path] }.keys.size
         puts "🌱 Seeded #{examples.size} examples across #{file_count} files"
       rescue Errno::ECONNREFUSED
@@ -140,6 +155,7 @@ module Specwrk
 
         start_workers
         wait_for_workers_exit
+        drain_outputs
 
         require "specwrk/cli_reporter"
         Specwrk::CLIReporter.new.report
@@ -180,9 +196,10 @@ module Specwrk
       include Servable
 
       desc "Start a server and workers, monitor until complete"
+      option :max_retries, default: 0, desc: "Number of times an example will be re-run should it fail"
       argument :dir, required: false, default: "spec", desc: "Relative spec directory to run against"
 
-      def call(dir:, **args)
+      def call(max_retries:, dir:, **args)
         self.class.setup(**args)
         $stdout.sync = true
 
@@ -204,6 +221,7 @@ module Specwrk
           require "specwrk/list_examples"
           require "specwrk/client"
 
+          ENV["SPECWRK_FORKED"] = "1"
           ENV["SPECWRK_SEED"] = "1"
           examples = ListExamples.new(dir).examples
 
@@ -211,7 +229,7 @@ module Specwrk
           Client.wait_for_server!
           status "Server responding ✓"
           status "Seeding #{examples.length} examples..."
-          Client.new.seed(examples)
+          Client.new.seed(examples, max_retries)
           file_count = examples.group_by { |e| e[:file_path] }.keys.size
           status "🌱 Seeded #{examples.size} examples across #{file_count} files"
         end
@@ -228,6 +246,7 @@ module Specwrk
         status "#{worker_count} workers started ✓\n"
         Specwrk.wait_for_pids_exit(@worker_pids)
 
+        drain_outputs
         return if Specwrk.force_quit
 
         require "specwrk/cli_reporter"

data/lib/specwrk/client.rb

@@ -127,8 +127,8 @@ module Specwrk
       retry
     end
 
-    def seed(examples)
-      response = post "/seed", body: examples.to_json
+    def seed(examples, max_retries)
+      response = post "/seed", body: {max_retries: max_retries, examples: examples}.to_json
 
       (response.code == "200") ? true : raise(UnhandledResponseError.new("#{response.code}: #{response.body}"))
     end

data/lib/specwrk/store.rb

@@ -99,6 +99,7 @@ module Specwrk
   class PendingStore < Store
     RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
     ORDER_KEY = :____order
+    MAX_RETRIES_KEY = :____max_retries
 
     def run_time_bucket_maximum=(val)
       @run_time_bucket_maximum = self[RUN_TIME_BUCKET_MAXIMUM_KEY] = val
@@ -122,6 +123,14 @@ module Specwrk
       @order ||= self[ORDER_KEY] || []
     end
 
+    def max_retries=(val)
+      @max_retries = self[MAX_RETRIES_KEY] = val
+    end
+
+    def max_retries
+      @max_retries ||= self[MAX_RETRIES_KEY] || 0
+    end
+
     def keys
       return super if order.length.zero?
 
@@ -130,7 +139,8 @@ module Specwrk
 
     def merge!(hash)
       super
-      self.order = hash.keys
+
+      self.order = order + (hash.keys - order)
     end
 
     def clear
@@ -140,6 +150,7 @@ module Specwrk
 
     def reload
       @order = nil
+      @max_retries = nil
       super
     end
 

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.13.1"
+  VERSION = "0.14.0"
 end

data/lib/specwrk/web/endpoints.rb

@@ -92,6 +92,10 @@ module Specwrk
           @completed ||= CompletedStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "completed"))
         end
 
+        def failure_counts
+          @failure_counts ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "failure_counts"))
+        end
+
         def metadata
           @metadata ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "metadata"))
         end
@@ -141,6 +145,10 @@ module Specwrk
 
         def with_response
           pending.clear
+          failure_counts.clear
+
+          pending.max_retries = payload.fetch(:max_retries, "0").to_i
+
           new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
           pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
 
@@ -154,10 +162,10 @@ module Specwrk
         def examples_with_run_times
           @examples_with_run_times ||= begin
             unsorted_examples_with_run_times = []
-            all_ids = payload.map { |example| example[:id] }
+            all_ids = payload[:examples].map { |example| example[:id] }
             all_run_times = run_times.multi_read(*all_ids)
 
-            payload.each do |example|
+            payload[:examples].each do |example|
               run_time = all_run_times[example[:id]]
 
               unsorted_examples_with_run_times << [example[:id], example.merge(expected_run_time: run_time)]
@@ -250,15 +258,52 @@ module Specwrk
 
         def with_response
           completed.merge!(completed_examples)
-          processing.delete(*completed_examples.keys)
+          processing.delete(*(completed_examples.keys + retry_examples.keys))
+          pending.merge!(retry_examples)
+          failure_counts.merge!(retry_examples_new_failure_counts)
 
           with_pop_response
         end
 
         private
 
+        def all_examples
+          @all_examples ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
+        end
+
         def completed_examples
-          @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
+          @completed_examples ||= all_examples.map do |id, example|
+            next if retry_example?(example)
+
+            [id, example]
+          end.compact.to_h
+        end
+
+        def retry_examples
+          @retry_examples ||= all_examples.map do |id, example|
+            next unless retry_example?(example)
+
+            [id, example]
+          end.compact.to_h
+        end
+
+        def retry_examples_new_failure_counts
+          @retry_examples_new_failure_counts ||= retry_examples.map do |id, _example|
+            [id, all_example_failure_counts.fetch(id, 0) + 1]
+          end.to_h
+        end
+
+        def retry_example?(example)
+          return false unless example[:status] == "failed"
+          return false unless pending.max_retries.positive?
+
+          example_failure_count = all_example_failure_counts.fetch(example[:id], 0)
+
+          example_failure_count < pending.max_retries
+        end
+
+        def all_example_failure_counts
+          @all_example_failure_counts ||= failure_counts.multi_read(*all_examples.keys)
         end
 
         def completed_examples_status_counts

data/lib/specwrk/worker.rb

@@ -54,7 +54,7 @@ module Specwrk
         end
       end
 
-      executor.final_output.tap(&:rewind).each_line { |line| $stdout.write line }
+      executor.final_output.tap(&:rewind).each_line { |line| final_output.write line }
 
       @heartbeat_thread.kill
       client.close
@@ -109,6 +109,10 @@ module Specwrk
 
     attr_reader :running, :client, :executor
 
+    def final_output
+      $final_output || $stdout # standard:disable Style/GlobalVars
+    end
+
     def status
       return 0 if @all_examples_completed && client.worker_status.zero?
       return 1 if Specwrk.force_quit

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.14.0
 platform: ruby
 authors:
 - Daniel Westendorf