specwrk 0.15.10 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1201e277f5b36c6ac9ff9ac20cf2b591f54d42822b071cf61b57a463164c5c4e
-  data.tar.gz: 0c53e4c86f48062680801f3d0de4ac5f691e55b3afaae98398750b9d8bca865d
+  metadata.gz: 1736573f9c3fc84992518a7a2ea486c03eefe4885b8b857a79b15551533deec0
+  data.tar.gz: 7106e3b4c856bfd9ff89e5fe562ba3da45ad5981356b295e24b55579aa48609f
 SHA512:
-  metadata.gz: 77142f3689fc39ec758869a42611d40b0f2013ebcba1a3c76a3ed4e2d7b6758f80ac0d9e2a93f58fcf58de9d3ada76ed34380d3a81dfc1bc2d1a49cab98fc67c
-  data.tar.gz: bf4c4f3d3784feaed50f41d84e52dac09d232e6fc9d14e79c00961f79286c25775e4a922c4512a0f17ffd45b4bc863ae61b8fe6a70ab9c233c033303e4c7245f
+  metadata.gz: 067222d93a38d847886d40b5a5121e24d68f633115ecfddccdd7c199adf598fd780cb8ca5fd1110028679828436ac3bc8a4335ca54c7a481aeaa7dc85938d0a4
+  data.tar.gz: 5d10c755bb71d7ace592e33c5fa8b2c4a2d9a349805870914ca0c9ac3410306765324211820c1283c8c2d8846ddbbcf5d1c88f89f2d5b89899f554a6e5c187dd

data/README.md

@@ -34,7 +34,7 @@ Commands:
 Intended for quick ad-hoc local host development or single-node CI runs. This command starts a queue server, seeds it with examples from the `spec/` directory, and starts `8` worker processes. It will report the ultimate success or failure.
 
 ```sh
-$ start --help
+$ specwrk start --help
 Command:
   specwrk start
 
@@ -52,6 +52,7 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN, default: "main"
   --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT, default: "5"
+  --network-retries=VALUE           # The number of times to retry in the event of a network failure. Overrides SPECWRK_NETWORK_RETRIES, default: "1"
   --id=VALUE                        # The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used
   --count=VALUE, -c VALUE           # The number of worker processes you want to start, default: 1
   --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
@@ -113,6 +114,7 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN, default: "main"
   --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT, default: "5"
+  --network-retries=VALUE           # The number of times to retry in the event of a network failure. Overrides SPECWRK_NETWORK_RETRIES, default: "1"
   --max-retries=VALUE               # Number of times an example will be re-run should it fail, default: 0
   --help, -h                        # Print this help
 ```
@@ -140,6 +142,7 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN, default: "main"
   --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT, default: "5"
+  --network-retries=VALUE           # The number of times to retry in the event of a network failure. Overrides SPECWRK_NETWORK_RETRIES, default: "1"
   --help, -h                        # Print this help
 ```
 
@@ -164,13 +167,13 @@ Options:
 ```
 
 ## Configuring your test environment
-If you test suite tracks state, starts servers, etc. and you plan on running many processes on the same node, you'll need to make
+If your test suite tracks state, starts servers, etc. and you plan on running many processes on the same node, you'll need to make
 adjustments to avoid conflicting port usage or database/state mutations.
 
 `specwrk` workers will have `TEST_ENV_NUMBER={i}` set to help you configure approriately.
 
 ### Rails
-Rails has had easy multi-process test setup for a while now by creating unique test databases per process. For my rails v7.2 app which uses PostgreSQL and Capyabara, I made these changes to my `spec/rails_helper.rb`:
+Rails has had easy multi-process test setup for a while now by creating unique test databases per process. For my rails v7.2 app which uses PostgreSQL and Capybara, I made these changes to my `spec/rails_helper.rb`:
 
 ```diff
 ++ if ENV["TEST_ENV_NUMBER"]
@@ -200,7 +203,7 @@ Make sure to persist `$SPECWRK_OUT/report.json` between runs so that subsequent
 [CircleCI Example](https://github.com/danielwestendorf/specwrk/blob/main/.circleci/config.yml) (specwrk-single-node job)
 
 ### Multi-node, multi-process
-Multi-node, multi-process works best when have many nodes running tests. This distributes the test execution across the nodes until the queue is for the run is empty, optimizing for slowest specs first. This distributes test execution across all nodes evenly(-ish).
+Multi-node, multi-process works best when you have many nodes running tests. This distributes the test execution across the nodes until the queue is for the run is empty, optimizing for slowest specs first. This distributes test execution across all nodes evenly(-ish).
 
 To accomplish this, a central queue server is required, examples must be explicitly seeded, and workers explicitly started.
 
@@ -221,7 +224,7 @@ Start a persistent Queue Server given one of the following methods
 
 ### Configuring your Queue Server
 - Secure your server with a key either with the `SPECWRK_SRV_KEY` environment variable or `--key` CLI option
-- Configure the server output to be a persisted volume so your timings survive between system restarts with the `SPECWRK_SRV_STORE_URI` environment variable or `--store-uri` CLI option. By default, `memory:///` will be used for the run's data stores (so run data will no survive server restarts) while `file://#{Dir.tmpdir}` will be used for run timings. Pass `--store-uri file:///whatever/absolute/path` to store all data on disk (required for multiple server processes).
+- Configure the server output to be a persisted volume so your timings survive between system restarts with the `SPECWRK_SRV_STORE_URI` environment variable or `--store-uri` CLI option. By default, `memory:///` will be used for the run's data stores (so run data will not survive server restarts) while `file://#{Dir.tmpdir}` will be used for run timings. Pass `--store-uri file:///whatever/absolute/path` to store all data on disk (required for multiple server processes).
 
 See [specwrk-store-redis_adapter](https://github.com/danielwestendorf/specwrk-store-redis_adapter) for Redis-compatible backed storage.
 
@@ -245,7 +248,7 @@ map(/_spec\.rb$/) do |spec_path|
   spec_path
 end
 
-# If a file in lib changes, map it to the spec folder for it's spec file
+# If a file in lib changes, map it to the spec folder for its spec file
 map(/lib\/.*\.rb$/) do |path|
   path.gsub(/lib\/(.+)\.rb/, "spec/\\1_spec.rb")
 end
@@ -255,7 +258,7 @@ end
 #   path.gsub(/app\/models\/(.+)\.rb/, "spec/models/\\1_spec.rb")
 # end
 #
-# If a controlelr file changes (assuming rails app structure), run the controller and system specs file
+# If a controller file changes (assuming rails app structure), run the controller and system specs file
 # map(/app\/controllers\/.*.rb$/) do |path|
 #   [
 #     path.gsub(/app\/controllers\/(.+)\.rb/, "spec/controllers/\\1_spec.rb"),
@@ -265,7 +268,7 @@ end
 ```
 
 ## Prior/other works
-There are many prior works for running rspec tests across multiple processes. Most of them combine process output making failures hard to grok. Some are good at running tests locally, but not on CI, while others are inversely true. Others are comercial or impactical without making a purchase.
+There are many prior works for running rspec tests across multiple processes. Most of them combine process output making failures hard to grok. Some are good at running tests locally, but not on CI, while others are inversely true. Others are commercial or impractical without making a purchase.
 
 specwrk is different because it:
 1. Puts your developer experience first. Easy execution. No messy outputs. Retries built in. Easy(er) debugging of flaky tests. 
@@ -287,7 +290,7 @@ specwrk is different because it:
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/dwestendorf/specwrk.
+Bug reports and pull requests are welcome on GitHub at https://github.com/danielwestendorf/specwrk.
 
 ## License
 

data/lib/specwrk/cli.rb

@@ -20,13 +20,15 @@ module Specwrk
         base.unique_option :key, type: :string, default: ENV.fetch("SPECWRK_SRV_KEY", ""), aliases: ["-k"], desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
         base.unique_option :run, type: :string, default: ENV.fetch("SPECWRK_RUN", "main"), aliases: ["-r"], desc: "The run identifier for this job execution. Overrides SPECWRK_RUN"
         base.unique_option :timeout, type: :integer, default: ENV.fetch("SPECWRK_TIMEOUT", "5"), aliases: ["-t"], desc: "The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT"
+        base.unique_option :network_retries, type: :integer, default: ENV.fetch("SPECWRK_NETWORK_RETRIES", "1"), desc: "The number of times to retry in the event of a network failure. Overrides SPECWRK_NETWORK_RETRIES"
       end
 
-      on_setup do |uri:, key:, run:, timeout:, **|
+      on_setup do |uri:, key:, run:, timeout:, network_retries:, **|
         ENV["SPECWRK_SRV_URI"] = uri
         ENV["SPECWRK_SRV_KEY"] = key
         ENV["SPECWRK_RUN"] = run
         ENV["SPECWRK_TIMEOUT"] = timeout
+        ENV["SPECWRK_NETWORK_RETRIES"] = network_retries
       end
     end
 

data/lib/specwrk/client.rb

@@ -115,16 +115,6 @@ module Specwrk
       else
         raise UnhandledResponseError.new("#{response.code}: #{response.body}")
       end
-    rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout => e
-      @retry_count ||= 0
-      @retry_count += 1
-
-      raise e if @retry_count == 5
-
-      warn e
-      sleep @retry_count
-
-      retry
     end
 
     def seed(examples, max_retries)
@@ -172,6 +162,16 @@ module Specwrk
           @worker_status = response["x-specwrk-status"].to_i if response["x-specwrk-status"]
         end
       end
+    rescue Net::ReadTimeout, Net::WriteTimeout => e
+      @retry_count ||= 0
+
+      raise e if @retry_count == ENV["SPECWRK_NETWORK_RETRIES"].to_i
+      @retry_count += 1
+
+      warn e
+      sleep @retry_count
+
+      retry
     end
 
     def default_headers

data/lib/specwrk/store.rb

@@ -105,6 +105,47 @@ module Specwrk
     end
   end
 
+  class WorkerStore < Store
+    FIRST_SEEN_AT_KEY = :____first_seen_at_key
+    LAST_SEEN_AT_KEY = :____last_seen_at_key
+
+    def first_seen_at=(val)
+      @first_seen_at = nil
+
+      self[FIRST_SEEN_AT_KEY] = val.to_i
+    end
+
+    def first_seen_at
+      @first_seen_at ||= begin
+        value = self[FIRST_SEEN_AT_KEY]
+        return @first_seen_at = value unless value
+
+        @first_seen_at = Time.at(value.to_i)
+      end
+    end
+
+    def last_seen_at=(val)
+      @last_seen_at = nil
+
+      self[LAST_SEEN_AT_KEY] = val.to_i
+    end
+
+    def last_seen_at
+      @last_seen_at ||= begin
+        value = self[LAST_SEEN_AT_KEY]
+        return @last_seen_at = value unless value
+
+        @last_seen_at = Time.at(value.to_i)
+      end
+    end
+
+    def reload
+      @last_seen_at = nil
+      @first_seen_at = nil
+      super
+    end
+  end
+
   class PendingStore < Store
     RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
     ORDER_KEY = :____order
@@ -234,22 +275,6 @@ module Specwrk
   end
 
   class ProcessingStore < Store
-    def expired
-      @expired ||= begin
-        bucket = []
-
-        keys.each_slice(24).each do |key_group|
-          examples = multi_read(*key_group)
-          examples.each do |id, example|
-            next if example[:completion_threshold].nil?
-
-            bucket << [id, example] if example[:completion_threshold] < Time.now.to_i
-          end
-        end
-
-        bucket.to_h
-      end
-    end
   end
 
   class CompletedStore < Store

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.15.10"
+  VERSION = "0.16.0"
 end

data/lib/specwrk/web/endpoints/base.rb

@@ -21,8 +21,8 @@ module Specwrk
 
           before_lock
 
-          worker[:first_seen_at] ||= Time.now.iso8601
-          worker[:last_seen_at] = Time.now.iso8601
+          worker.first_seen_at ||= Time.now
+          worker.last_seen_at = Time.now
 
           final_response = with_lock do
             started_at = metadata[:started_at] ||= Time.now.iso8601
@@ -109,7 +109,11 @@ module Specwrk
         end
 
         def worker
-          @worker ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "workers", request.get_header("HTTP_X_SPECWRK_ID").to_s))
+          @worker ||= worker_store_for(worker_id)
+        end
+
+        def worker_id
+          request.get_header("HTTP_X_SPECWRK_ID").to_s
         end
 
         def worker_status
@@ -118,6 +122,10 @@ module Specwrk
           worker[:failed] || 1
         end
 
+        def worker_store_for(id)
+          WorkerStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "workers", id))
+        end
+
         def run_id
           request.get_header("HTTP_X_SPECWRK_RUN")
         end

data/lib/specwrk/web/endpoints/popable.rb

@@ -15,9 +15,9 @@ module Specwrk
             [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
           elsif completed.any? && processing.empty?
             [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
-          elsif processing.any? && processing.expired.keys.any?
-            pending.merge!(processing.expired)
-            processing.delete(*processing.expired.keys)
+          elsif expired_examples.length.positive?
+            pending.merge!(expired_examples.each { |_id, example| example[:worker_id] = worker_id })
+            processing.delete(*expired_examples.keys)
             @examples = nil
 
             [200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
@@ -29,13 +29,10 @@ module Specwrk
         def examples
           @examples ||= begin
             examples = pending.shift_bucket
-            bucket_run_time_total = examples.map { |example| example.fetch(:expected_run_time, 10.0) }.compact.sum * 2
-            maximum_completion_threshold = (pending.run_time_bucket_maximum || 30.0) * 2
-            completion_threshold = Time.now + [bucket_run_time_total, maximum_completion_threshold, 20.0].max
 
             processing_data = examples.map do |example|
               [
-                example[:id], example.merge(completion_threshold: completion_threshold.to_f)
+                example[:id], example.merge(worker_id: worker_id, processing_started_at: Time.now.to_i)
               ]
             end
 
@@ -44,6 +41,27 @@ module Specwrk
             examples
           end
         end
+
+        def expired_examples
+          return unless processing.any?
+
+          @expired_examples ||= processing.to_h.select { |_id, example| expired?(example) }
+        end
+
+        # Has the worker missed two heartbeat check-ins?
+        def expired?(example)
+          return false unless example[:worker_id]
+          return false unless example[:processing_started_at]
+          return false unless example[:processing_started_at] < (Time.now - 20).to_i
+
+          workers_last_heartbeats[example[:worker_id]] < Time.now - 20
+        end
+
+        def workers_last_heartbeats
+          @workers_last_heartbeats ||= Hash.new do |h, k|
+            h[k] = worker_store_for(k).last_seen_at || Time.at(0)
+          end
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.15.10
+  version: 0.16.0
 platform: ruby
 authors:
 - Daniel Westendorf