specwrk 0.19.1 → 0.19.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86a495d990068d402539acc8e3e06ba3380d49f2c7f647c4ddb9359e8c6644f3
-  data.tar.gz: 723d7ade3f6fde1fef61dd523611f73b5791885fbfec529f79f1113fdccb6802
+  metadata.gz: b2eb07bead54ad609b2d8fb821793a60b783485326daeccc092c14343ae96dc0
+  data.tar.gz: c41e008f026a61f2748cc8f5cd9c7dc2c257103518aa89fded4c84c9bd2ca976
 SHA512:
-  metadata.gz: f892c6d698d85db59d06b3372d7f7a41ce8011f38c3343e9ce41ee41e5592c0ea034686242c94a928433eddb878211509a72231bd407e8f346498b95ec31ac19
-  data.tar.gz: d9af25e6db969725369118561c2d3e1ec37c86a3cb26c054b6abba3a45b2617076afdd3b66feb9ed668b988a0f8ed888e007cb5ec24eee35318b50084c1987a5
+  metadata.gz: 1f9a7206df73023a2262d9775ff3b1983f403b4630ce54a04f7c4500ab77c51d30df6262604332b37e37fdf21cb98fe39952fcd0287f8b9ded7811a39ac97afe
+  data.tar.gz: ffbf1b2583044b4f3e4bdbc7cadb35278f93eb64274905c5869398f5ece9d2854ddad0c75463ac8172d025d8dcc7a92c8486d5d9ef581ad269c875aca5c22420

data/.standard.yml

@@ -1,3 +1,3 @@
 # For available configuration options, see:
 #   https://github.com/standardrb/standard
-ruby_version: 3.1
+ruby_version: 3.4

data/CHANGELOG.md

@@ -1,5 +1,75 @@
 # Changelog
 
+## Unreleased  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.19.3...main)  
+
+## v0.19.3 — 2026-07-26  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.19.2...v0.19.3)  
+- Retry transient file store read descriptor errors — [@danielwestendorf](https://github.com/danielwestendorf)  
+- Complete omitted worker processing examples as failures — [@danielwestendorf](https://github.com/danielwestendorf), [@benjaminwood](https://github.com/benjaminwood)  
+
+
+## v0.19.2 — 2025-12-09  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.19.1...v0.19.2)  
+- Optimize msgpack usage by using one packer/unpacker — [#180](https://github.com/danielwestendorf/specwrk/pull/180) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Standard style to v3.4 — [#179](https://github.com/danielwestendorf/specwrk/pull/179) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Add msgpack to the docker server — [#178](https://github.com/danielwestendorf/specwrk/pull/178) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.19.1 — 2025-12-08  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.19.0...v0.19.1)  
+- Add missing require for serializer in base adapter — [#177](https://github.com/danielwestendorf/specwrk/pull/177) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.19.0 — 2025-12-08  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.18.0...v0.19.0)  
+- Add support for configured serializer, adding msgpack — [#176](https://github.com/danielwestendorf/specwrk/pull/176) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.18.0 — 2025-12-07  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.17.1...v0.18.0)  
+- Spawn workers with `spawn` instead of `fork` — [#174](https://github.com/danielwestendorf/specwrk/pull/174) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- If pending is empty, return before acquiring lock — [#173](https://github.com/danielwestendorf/specwrk/pull/173) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Fix typo resulting in wrong group by — [#166](https://github.com/danielwestendorf/specwrk/pull/166) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.17.1 — 2025-12-06  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.17.0...v0.17.1)  
+- Version bump only (no functional changes) — [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.17.0 — 2025-12-06  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.16.3...v0.17.0)  
+- Add bucket store — [#170](https://github.com/danielwestendorf/specwrk/pull/170) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- s/Bucket/BucketStore — [#171](https://github.com/danielwestendorf/specwrk/pull/171) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Bucket at seed — [#172](https://github.com/danielwestendorf/specwrk/pull/172) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Split Store classes into their own files — [#169](https://github.com/danielwestendorf/specwrk/pull/169) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Fix worker failing to flush on worker exit — [#167](https://github.com/danielwestendorf/specwrk/pull/167) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Typo — [#168](https://github.com/danielwestendorf/specwrk/pull/168) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.16.3 — 2025-12-01  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.16.2...v0.16.3)  
+- Disable locks on heartbeat and health endpoints — [#163](https://github.com/danielwestendorf/specwrk/pull/163) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Add specwrk worker info to logger — [#164](https://github.com/danielwestendorf/specwrk/pull/164) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Ignore Agents.md — [#165](https://github.com/danielwestendorf/specwrk/pull/165) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.16.2 — 2025-10-23  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.16.1...v0.16.2)  
+- exit(1) from seed process when example size is zero — [#158](https://github.com/danielwestendorf/specwrk/pull/158) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.16.1 — 2025-10-03  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.16.0...v0.16.1)  
+- Ensure next_examples are always cleared after execution — [#156](https://github.com/danielwestendorf/specwrk/pull/156) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- More frequent heartbeats — [#154](https://github.com/danielwestendorf/specwrk/pull/154) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Ensure mutex is held when a lock is expected — [#153](https://github.com/danielwestendorf/specwrk/pull/153) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.16.0 — 2025-10-02  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.15.10...v0.16.0)  
+- Support retry of all client requests, configurable count — [#151](https://github.com/danielwestendorf/specwrk/pull/151) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Expire workers from worker heartbeats — [#149](https://github.com/danielwestendorf/specwrk/pull/149) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Fix a few minor typos in README — [#147](https://github.com/danielwestendorf/specwrk/pull/147) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Run flake test if the srv uri is localhost — [#146](https://github.com/danielwestendorf/specwrk/pull/146) by [@danielwestendorf](https://github.com/danielwestendorf)  
+
+## v0.15.10 — 2025-08-28  
+[Compare](https://github.com/danielwestendorf/specwrk/compare/v0.15.9...v0.15.10)  
+- Handle default path for start and seed better — [#144](https://github.com/danielwestendorf/specwrk/pull/144) by [@danielwestendorf](https://github.com/danielwestendorf)  
+- Mention Redis storage adapter — [@danielwestendorf](https://github.com/danielwestendorf)  
+
 ## v0.15.9 — 2025-08-28  
 [Compare](https://github.com/danielwestendorf/specwrk/compare/v0.15.8...v0.15.9)  
 - Support lock skipping for read-only endpoints — [#139](https://github.com/danielwestendorf/specwrk/pull/139) by [@danielwestendorf](https://github.com/danielwestendorf)  

data/docker/Dockerfile.server

@@ -14,7 +14,7 @@ COPY $GEM_FILE ./
 RUN gem install ./$GEM_FILE --no-document
 RUN rm ./$GEM_FILE
 
-RUN gem install pitchfork thruster specwrk-store-redis_adapter
+RUN gem install pitchfork thruster specwrk-store-redis_adapter msgpack
 COPY config.ru ./
 COPY docker/pitchfork.conf ./
 

data/lib/specwrk/store/file_adapter.rb

@@ -10,6 +10,9 @@ require "specwrk/store/base_adapter"
 module Specwrk
   class Store
     class FileAdapter < BaseAdapter
+      READ_ATTEMPTS = 3
+      READ_RETRY_DELAY = 0.01
+
       @work_queue = Queue.new
       @threads = []
 
@@ -91,18 +94,21 @@ module Specwrk
 
         read_keys.each do |key|
           self.class.schedule_work do
-            result_queue.push([key.to_s, read(key)])
+            result_queue.push([:ok, key.to_s, read(key)])
+          rescue => e
+            result_queue.push([:error, key.to_s, e])
           end
         end
 
-        Thread.pass until result_queue.length == read_keys.length
+        raw_results = read_keys.length.times.map { result_queue.pop }
+        error = raw_results.find { |status, _key, _result| status == :error }
+        raise error.last if error
 
         results = {}
-        until result_queue.empty?
-          result = result_queue.pop
-          next if result.last.nil?
+        raw_results.each do |_status, key, content|
+          next if content.nil?
 
-          results[result.first] = self.class.serializer.load(result.last)
+          results[key] = self.class.serializer.load(content)
         end
 
         read_keys.map { |key| [key.to_s, results[key.to_s]] if results.key?(key.to_s) }.compact.to_h # respect order requested in the returned hash
@@ -140,7 +146,17 @@ module Specwrk
 
       def read(key)
         filename = filename_for_key key
-        File.binread(filename)
+        attempts = 0
+
+        begin
+          attempts += 1
+          File.binread(filename)
+        rescue Errno::EBADF
+          raise if attempts >= READ_ATTEMPTS
+
+          sleep(READ_RETRY_DELAY * attempts)
+          retry
+        end
       rescue Errno::ENOENT
         nil
       end

data/lib/specwrk/store/serializer.rb

@@ -22,31 +22,43 @@ module Specwrk
       end
 
       module MessagePack
-        module_function
-
         def ensure_loaded!
+          return if defined?(::MessagePack)
+
           require "msgpack"
         rescue LoadError
           raise LoadError, "Unable to use msgpack, gem not found. Add `gem 'msgpack' to your Gemfile and bundle install"
         end
 
-        module_function
-
         def adapter_name
           "msgpack"
         end
 
         def dump(value)
+          packer.tap(&:clear)
+            .tap { |msgpack| msgpack.write(value) }
+            .then(&:to_s)
+        end
+
+        def load(payload)
+          unpacker.tap(&:reset)
+            .tap { |msgpack| msgpack.feed(payload) }
+            .then { |msgpack| msgpack.read }
+        end
+
+        def packer
           ensure_loaded!
 
-          ::MessagePack.dump(value)
+          Thread.current[:__specwrk_msgpack_packer] ||= ::MessagePack::Packer.new
         end
 
-        def load(payload)
+        def unpacker
           ensure_loaded!
 
-          ::MessagePack.load(payload, symbolize_keys: true)
+          Thread.current[:__specwrk_msgpack_unpacker] ||= ::MessagePack::Unpacker.new(symbolize_keys: true)
         end
+
+        module_function :ensure_loaded!, :adapter_name, :dump, :load, :packer, :unpacker
       end
     end
 

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.19.1"
+  VERSION = "0.19.3"
 end

data/lib/specwrk/web/endpoints/complete_and_pop.rb

@@ -7,12 +7,14 @@ module Specwrk
     module Endpoints
       class CompleteAndPop < Popable
         EXAMPLE_STATUSES = %w[passed failed pending]
+        SYNTHETIC_FAILURE_CLASS = "Specwrk::WorkerExecutionError"
 
         def with_response
           retry_examples # pre-calculate before lock
+          omitted_processing_examples # pre-calculate before lock
 
           with_lock do
-            processing.delete(*(completed_examples.keys + retry_examples.keys))
+            processing.delete(*(completed_examples.keys + retry_examples.keys).map(&:to_s))
             pending.merge!(retry_examples)
           end
 
@@ -26,24 +28,30 @@ module Specwrk
 
         private
 
-        def all_examples
-          @all_examples ||= payload.map { |example| [example[:id], example] if processing_examples[example[:id]] }.compact.to_h
+        def reported_processing_examples
+          @reported_processing_examples ||= payload.map do |example|
+            [example[:id], example] if processing_examples_for_payload[example[:id]]
+          end.compact.to_h
         end
 
-        def processing_examples
-          @processing_examples ||= processing.multi_read(*payload.map { |example| example[:id] })
+        def processing_examples_for_payload
+          @processing_examples_for_payload ||= processing.multi_read(*payload.map { |example| example[:id] })
         end
 
         def completed_examples
-          @completed_examples ||= all_examples.map do |id, example|
-            next if retry_example?(example)
+          @completed_examples ||= begin
+            reported_completed_examples = reported_processing_examples.map do |id, example|
+              next if retry_example?(example)
 
-            [id, example]
-          end.compact.to_h
+              [id, example]
+            end.compact.to_h
+
+            reported_completed_examples.merge(unreported_completed_examples)
+          end
         end
 
         def retry_examples
-          @retry_examples ||= all_examples.map do |id, example|
+          @retry_examples ||= reported_processing_examples.map do |id, example|
             next unless retry_example?(example)
 
             [id, example]
@@ -66,7 +74,48 @@ module Specwrk
         end
 
         def all_example_failure_counts
-          @all_example_failure_counts ||= failure_counts.multi_read(*all_examples.keys)
+          @all_example_failure_counts ||= failure_counts.multi_read(*reported_processing_examples.keys)
+        end
+
+        def omitted_processing_examples
+          @omitted_processing_examples ||= processing.to_h.select do |_id, example|
+            example[:worker_id].to_s == worker_id && !payload_example_ids.include?(example[:id].to_s)
+          end
+        end
+
+        def unreported_completed_examples
+          @unreported_completed_examples ||= omitted_processing_examples.transform_values { |example| synthetic_failure_for(example) }
+        end
+
+        def payload_example_ids
+          @payload_example_ids ||= payload.map { |example| example[:id].to_s }.uniq
+        end
+
+        def synthetic_failure_for(example)
+          finished_at = Time.now
+
+          {
+            id: example[:id],
+            full_description: "Specwrk worker execution failed before completing #{example[:id]}",
+            status: "failed",
+            file_path: example[:file_path],
+            line_number: line_number_for(example),
+            started_at: finished_at.iso8601(6),
+            finished_at: finished_at.iso8601(6),
+            run_time: 0.0,
+            exception: {
+              class: SYNTHETIC_FAILURE_CLASS,
+              message: "Worker #{worker_id} claimed this example but did not submit a completion result. Check worker stderr/stdout for the underlying RSpec output.",
+              backtrace: []
+            }
+          }
+        end
+
+        def line_number_for(example)
+          return example[:line_number] if example[:line_number]
+
+          match = example[:id].to_s.match(/:(\d+)\z/)
+          match[1].to_i if match
         end
 
         def completed_examples_status_counts
@@ -87,7 +136,9 @@ module Specwrk
         end
 
         def run_time_data
-          @run_time_data ||= payload.map { |example| [example[:id], example[:run_time]] }.to_h
+          @run_time_data ||= payload.map { |example| [example[:id], example[:run_time]] }.to_h.merge(
+            unreported_completed_examples.transform_values { |example| example[:run_time] }
+          )
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.19.1
+  version: 0.19.3
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -274,7 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Parallel rspec test runner from a queue of pending jobs.
 test_files: []