specwrk 0.16.3 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fdc41df570bf10c5122524e14a03c069e9896882c520207f2f7dbff38204e90
-  data.tar.gz: 2885ad26550da39ed2c350cf0c05e566a3160e9fa0f1d6f474a285b307153640
+  metadata.gz: 2ef3257927cc46b8361f2fe0ee98db82890b5f8a25882045e122317c2b223ccd
+  data.tar.gz: 8dac946fb5a4e8e673e4fef15eab5b4f16e1ffb4959688c1c6af967b9c06b34d
 SHA512:
-  metadata.gz: b230b21300808fe6bca6f13b51935ea471acdcbd68907283b9895d7757f96a79124dfe0654ea70573be7bd658b89026b443cb93d9e9558643fc8f7cd0f7ebea9
-  data.tar.gz: ac9a20a085d5b854100f5136f9eb7269a875ea6aa133e71107d75b498adb176640107d8fe14b8ebdff195f717cae7f56edcaf35993075035ce709865efeff518
+  metadata.gz: 5c9ea3eb229ba95a345f981ad5bcb9faa9998094b4c9ca29ea310756a6d746670a7dfbe3bcca29a01eb8067073795bc3e8a7dead802f8bf1e8263a63f8217225
+  data.tar.gz: 938de3505dcb4ac54daecadb4b00eea0ffda3c3a3868e85341334dc8e4f11be570f64c241f433fba68415ce22afe791b54e8222d0a771c7c838b207afcbd6dae

data/Specwrk.watchfile.rb

@@ -16,7 +16,7 @@ end
 #   path.gsub(/app\/models\/(.+)\.rb/, "spec/models/\\1_spec.rb")
 # end
 #
-# If a controlelr file changes (assuming rails app structure), run the controller and system specs file
+# If a controller file changes (assuming rails app structure), run the controller and system specs file
 # map(/app\/controllers\/.*.rb$/) do |path|
 #   [
 #     path.gsub(/app\/controllers\/(.+)\.rb/, "spec/controllers/\\1_spec.rb"),

data/lib/specwrk/store/base.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Specwrk
+  class Store
+    class << self
+      def with_lock(uri, key)
+        adapter_klass(uri).with_lock(uri, key) { yield }
+      end
+
+      def adapter_klass(uri)
+        case uri.scheme
+        when "memory"
+          require "specwrk/store/memory_adapter" unless defined?(MemoryAdapter)
+
+          MemoryAdapter
+        when "file"
+          require "specwrk/store/file_adapter" unless defined?(FileAdapter)
+
+          FileAdapter
+        when /redis/
+          begin
+            require "specwrk/store/redis_adapter" unless defined?(RedisAdapter)
+          rescue LoadError
+            warn "Unable use RedisAdapter with #{uri}, gem not found. Add `gem 'specwrk-store-redis_adapter'` to your Gemfile and bundle install."
+            exit(1)
+          end
+
+          RedisAdapter
+        end
+      end
+    end
+
+    def initialize(uri_string, scope)
+      @uri = URI(uri_string)
+      @scope = scope
+    end
+
+    def [](key)
+      adapter[key.to_s]
+    end
+
+    def multi_read(*keys)
+      adapter.multi_read(*keys)
+    end
+
+    def []=(key, value)
+      adapter[key.to_s] = value
+    end
+
+    def keys
+      all_keys = adapter.keys
+
+      all_keys.reject { |k| k.start_with? "____" }
+    end
+
+    def length
+      keys.length
+    end
+
+    def any?
+      !empty?
+    end
+
+    def empty?
+      adapter.empty?
+    end
+
+    def delete(*keys)
+      adapter.delete(*keys)
+    end
+
+    def merge!(h2)
+      h2.transform_keys!(&:to_s)
+      adapter.merge!(h2)
+    end
+
+    def clear
+      adapter.clear
+    end
+
+    def to_h
+      multi_read(*keys).transform_keys!(&:to_sym)
+    end
+
+    def inspect
+      reload.to_h.dup
+    end
+
+    # Bypass any cached values. Helpful when you have two instances
+    # of the same store where one mutates data and the other needs to check
+    # on the status of that data (i.e. endpoint tests)
+    def reload
+      @adapter = nil
+      self
+    end
+
+    private
+
+    attr_reader :uri, :scope
+
+    def adapter
+      @adapter ||= self.class.adapter_klass(uri).new uri, scope
+    end
+  end
+end

data/lib/specwrk/store/bucket_store.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "specwrk/store/base"
+
+module Specwrk
+  class BucketStore < Store
+    EXAMPLES_KEY = :____examples
+
+    def examples=(val)
+      @examples = nil
+
+      self[EXAMPLES_KEY] = if val.nil? || val.length.zero?
+        nil
+      else
+        val
+      end
+    end
+
+    def examples
+      @examples ||= self[EXAMPLES_KEY] || []
+    end
+
+    def clear
+      @examples = nil
+      super
+    end
+
+    def reload
+      @examples = nil
+      super
+    end
+  end
+end

data/lib/specwrk/store/completed_store.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "time"
+require "specwrk/store/base"
+
+module Specwrk
+  class CompletedStore < Store
+    def dump
+      @run_times = []
+      @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
+      @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
+
+      @output = {
+        file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
+        meta: {failures: 0, passes: 0, pending: 0},
+        examples: {}
+      }
+
+      to_h.values.each { |example| calculate(example) }
+
+      @output[:meta][:total_run_time] = @run_times.sum
+      @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
+      @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
+      @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
+
+      @output
+    end
+
+    private
+
+    def calculate(example)
+      @run_times << example[:run_time]
+      @output[:file_totals][example[:file_path]] += example[:run_time]
+
+      started_at = Time.parse(example[:started_at])
+      finished_at = Time.parse(example[:finished_at])
+
+      @first_started_at = started_at if started_at < @first_started_at
+      @last_finished_at = finished_at if finished_at > @last_finished_at
+
+      case example[:status]
+      when "passed"
+        @output[:meta][:passes] += 1
+      when "failed"
+        @output[:meta][:failures] += 1
+      when "pending"
+        @output[:meta][:pending] += 1
+      end
+
+      @output[:examples][example[:id]] = example
+    end
+  end
+end

data/lib/specwrk/store/pending_store.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+require "specwrk/store/base"
+require "specwrk/store/bucket_store"
+
+module Specwrk
+  class PendingStore < Store
+    RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
+    MAX_RETRIES_KEY = :____max_retries
+    BUCKET_IDS_KEY = :____bucket_ids
+
+    def run_time_bucket_maximum=(val)
+      @run_time_bucket_maximum = self[RUN_TIME_BUCKET_MAXIMUM_KEY] = val
+    end
+
+    def run_time_bucket_maximum
+      @run_time_bucket_maximum ||= self[RUN_TIME_BUCKET_MAXIMUM_KEY]
+    end
+
+    def max_retries=(val)
+      @max_retries = self[MAX_RETRIES_KEY] = val
+    end
+
+    def max_retries
+      @max_retries ||= self[MAX_RETRIES_KEY] || 0
+    end
+
+    def bucket_ids=(val)
+      @bucket_ids = nil
+
+      self[BUCKET_IDS_KEY] = if val.nil? || val.length.zero?
+        nil
+      else
+        val
+      end
+    end
+
+    def bucket_ids
+      @bucket_ids ||= self[BUCKET_IDS_KEY] || []
+    end
+
+    def merge!(hash)
+      return self if hash.nil? || hash.empty?
+
+      buckets = grouped_examples(hash.values)
+      new_bucket_ids = buckets.map { |examples| write_bucket(examples) }
+
+      self.bucket_ids = bucket_ids + new_bucket_ids
+      self
+    end
+
+    def clear
+      bucket_ids.each { |bucket_id| delete_bucket(bucket_id) }
+      @bucket_ids = nil
+
+      super
+    end
+
+    def reload
+      @max_retries = nil
+      @bucket_ids = nil
+      super
+    end
+
+    def shift_bucket
+      return nil if bucket_ids.empty?
+
+      bucket_id = bucket_ids.first
+      self.bucket_ids = bucket_ids.drop(1)
+      bucket_id
+    end
+
+    def push_examples(examples)
+      return self if examples.nil? || examples.empty?
+
+      new_bucket_id = write_bucket(examples)
+      self.bucket_ids = bucket_ids + [new_bucket_id]
+      self
+    end
+
+    def bucket_store_for(bucket_id)
+      BucketStore.new(uri.to_s, File.join(scope, "buckets", bucket_id))
+    end
+
+    def delete_bucket(bucket_id)
+      bucket_store_for(bucket_id).clear
+    end
+
+    def keys
+      bucket_ids
+    end
+
+    def length
+      bucket_ids.length
+    end
+
+    private
+
+    def write_bucket(examples)
+      bucket_id = SecureRandom.uuid
+      bucket_store_for(bucket_id).examples = examples
+      bucket_id
+    end
+
+    def grouped_examples(examples)
+      return [] if examples.empty?
+
+      examples_to_group = examples.dup
+
+      case grouping_strategy
+      when :file
+        group_by_file(examples_to_group)
+      else
+        group_by_timings(examples_to_group)
+      end
+    end
+
+    # Take consecutive examples with the same file_path
+    def group_by_file(examples)
+      buckets = []
+
+      examples.each do |example|
+        current_bucket = buckets.last
+
+        if current_bucket.nil? || current_bucket.first[:file_path] != example[:file_path]
+          buckets << [example]
+        else
+          current_bucket << example
+        end
+      end
+
+      buckets
+    end
+
+    # Take elements until the average runtime bucket has filled
+    def group_by_timings(examples)
+      buckets = []
+      return group_by_file(examples) unless run_time_bucket_maximum&.positive?
+
+      estimated_run_time_total = 0
+      current_bucket = []
+
+      examples.each do |example|
+        estimated_run_time_total += example[:expected_run_time] || run_time_bucket_maximum
+
+        if estimated_run_time_total > run_time_bucket_maximum && current_bucket.length.positive?
+          buckets << current_bucket
+          current_bucket = [example]
+          estimated_run_time_total = example[:expected_run_time] || run_time_bucket_maximum
+          next
+        end
+
+        current_bucket << example
+      end
+
+      buckets << current_bucket if current_bucket.any?
+      buckets
+    end
+
+    def grouping_strategy
+      return :file unless run_time_bucket_maximum&.positive?
+
+      (ENV["SPECWRK_SRV_GROUP_BY"] == "file") ? :file : :timings
+    end
+  end
+end

data/lib/specwrk/store/processing_store.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "specwrk/store/base"
+
+module Specwrk
+  class ProcessingStore < Store
+  end
+end

data/lib/specwrk/store/worker_store.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "time"
+require "specwrk/store/base"
+
+module Specwrk
+  class WorkerStore < Store
+    FIRST_SEEN_AT_KEY = :____first_seen_at_key
+    LAST_SEEN_AT_KEY = :____last_seen_at_key
+
+    def first_seen_at=(val)
+      @first_seen_at = nil
+
+      self[FIRST_SEEN_AT_KEY] = val.to_i
+    end
+
+    def first_seen_at
+      @first_seen_at ||= begin
+        value = self[FIRST_SEEN_AT_KEY]
+        return @first_seen_at = value unless value
+
+        @first_seen_at = Time.at(value.to_i)
+      end
+    end
+
+    def last_seen_at=(val)
+      @last_seen_at = nil
+
+      self[LAST_SEEN_AT_KEY] = val.to_i
+    end
+
+    def last_seen_at
+      @last_seen_at ||= begin
+        value = self[LAST_SEEN_AT_KEY]
+        return @last_seen_at = value unless value
+
+        @last_seen_at = Time.at(value.to_i)
+      end
+    end
+
+    def reload
+      @last_seen_at = nil
+      @first_seen_at = nil
+      super
+    end
+  end
+end

data/lib/specwrk/store.rb

@@ -1,326 +1,8 @@
 # frozen_string_literal: true
 
-require "time"
-
-module Specwrk
-  class Store
-    class << self
-      def with_lock(uri, key)
-        adapter_klass(uri).with_lock(uri, key) { yield }
-      end
-
-      def adapter_klass(uri)
-        case uri.scheme
-        when "memory"
-          require "specwrk/store/memory_adapter" unless defined?(MemoryAdapter)
-
-          MemoryAdapter
-        when "file"
-          require "specwrk/store/file_adapter" unless defined?(FileAdapter)
-
-          FileAdapter
-        when /redis/
-          begin
-            require "specwrk/store/redis_adapter" unless defined?(RedisAdapter)
-          rescue LoadError
-            warn "Unable use RedisAdapter with #{uri}, gem not found. Add `gem 'specwrk-store-redis_adapter'` to your Gemfile and bundle install."
-            exit(1)
-          end
-
-          RedisAdapter
-        end
-      end
-    end
-
-    def initialize(uri_string, scope)
-      @uri = URI(uri_string)
-      @scope = scope
-    end
-
-    def [](key)
-      adapter[key.to_s]
-    end
-
-    def multi_read(*keys)
-      adapter.multi_read(*keys)
-    end
-
-    def []=(key, value)
-      adapter[key.to_s] = value
-    end
-
-    def keys
-      all_keys = adapter.keys
-
-      all_keys.reject { |k| k.start_with? "____" }
-    end
-
-    def length
-      keys.length
-    end
-
-    def any?
-      !empty?
-    end
-
-    def empty?
-      adapter.empty?
-    end
-
-    def delete(*keys)
-      adapter.delete(*keys)
-    end
-
-    def merge!(h2)
-      h2.transform_keys!(&:to_s)
-      adapter.merge!(h2)
-    end
-
-    def clear
-      adapter.clear
-    end
-
-    def to_h
-      multi_read(*keys).transform_keys!(&:to_sym)
-    end
-
-    def inspect
-      reload.to_h.dup
-    end
-
-    # Bypass any cached values. Helpful when you have two instances
-    # of the same store where one mutates data and the other needs to check
-    # on the status of that data (i.e. endpoint tests)
-    def reload
-      @adapter = nil
-      self
-    end
-
-    private
-
-    attr_reader :uri, :scope
-
-    def adapter
-      @adapter ||= self.class.adapter_klass(uri).new uri, scope
-    end
-  end
-
-  class WorkerStore < Store
-    FIRST_SEEN_AT_KEY = :____first_seen_at_key
-    LAST_SEEN_AT_KEY = :____last_seen_at_key
-
-    def first_seen_at=(val)
-      @first_seen_at = nil
-
-      self[FIRST_SEEN_AT_KEY] = val.to_i
-    end
-
-    def first_seen_at
-      @first_seen_at ||= begin
-        value = self[FIRST_SEEN_AT_KEY]
-        return @first_seen_at = value unless value
-
-        @first_seen_at = Time.at(value.to_i)
-      end
-    end
-
-    def last_seen_at=(val)
-      @last_seen_at = nil
-
-      self[LAST_SEEN_AT_KEY] = val.to_i
-    end
-
-    def last_seen_at
-      @last_seen_at ||= begin
-        value = self[LAST_SEEN_AT_KEY]
-        return @last_seen_at = value unless value
-
-        @last_seen_at = Time.at(value.to_i)
-      end
-    end
-
-    def reload
-      @last_seen_at = nil
-      @first_seen_at = nil
-      super
-    end
-  end
-
-  class PendingStore < Store
-    RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
-    ORDER_KEY = :____order
-    MAX_RETRIES_KEY = :____max_retries
-
-    def run_time_bucket_maximum=(val)
-      @run_time_bucket_maximum = self[RUN_TIME_BUCKET_MAXIMUM_KEY] = val
-    end
-
-    def run_time_bucket_maximum
-      @run_time_bucket_maximum ||= self[RUN_TIME_BUCKET_MAXIMUM_KEY]
-    end
-
-    def order=(val)
-      @order = nil
-
-      self[ORDER_KEY] = if val.nil? || val.length.zero?
-        nil
-      else
-        val
-      end
-    end
-
-    def order
-      @order ||= self[ORDER_KEY] || []
-    end
-
-    def max_retries=(val)
-      @max_retries = self[MAX_RETRIES_KEY] = val
-    end
-
-    def max_retries
-      @max_retries ||= self[MAX_RETRIES_KEY] || 0
-    end
-
-    def keys
-      return super if order.length.zero?
-
-      order
-    end
-
-    def merge!(hash)
-      super
-
-      self.order = order + (hash.keys - order)
-    end
-
-    def clear
-      @order = nil
-      super
-    end
-
-    def reload
-      @order = nil
-      @max_retries = nil
-      super
-    end
-
-    def shift_bucket
-      return bucket_by_file unless run_time_bucket_maximum&.positive?
-
-      case ENV["SPECWRK_SRV_GROUP_BY"]
-      when "file"
-        bucket_by_file
-      else
-        bucket_by_timings
-      end
-    end
-
-    private
-
-    # Take elements from the hash where the file_path is the same
-    # Expects that the examples were merged in order of filename
-    def bucket_by_file
-      bucket = []
-      consumed_keys = []
-
-      all_keys = keys
-      key = all_keys.first
-      return [] if key.nil?
-
-      file_path = self[key][:file_path]
-
-      catch(:full) do
-        all_keys.each_slice(24).each do |key_group|
-          examples = multi_read(*key_group)
-
-          examples.each do |key, example|
-            throw :full unless example[:file_path] == file_path
-
-            bucket << example
-            consumed_keys << key
-          end
-        end
-      end
-
-      delete(*consumed_keys)
-      self.order = order - consumed_keys
-      bucket
-    end
-
-    # Take elements from the hash until the average runtime bucket has filled
-    def bucket_by_timings
-      bucket = []
-      consumed_keys = []
-
-      estimated_run_time_total = 0
-
-      catch(:full) do
-        keys.each_slice(24).each do |key_group|
-          examples = multi_read(*key_group)
-
-          examples.each do |key, example|
-            estimated_run_time_total += example[:expected_run_time] || run_time_bucket_maximum
-            throw :full if estimated_run_time_total > run_time_bucket_maximum && bucket.length.positive?
-
-            bucket << example
-            consumed_keys << key
-          end
-        end
-      end
-
-      delete(*consumed_keys)
-      self.order = order - consumed_keys
-      bucket
-    end
-  end
-
-  class ProcessingStore < Store
-  end
-
-  class CompletedStore < Store
-    def dump
-      @run_times = []
-      @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
-      @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
-
-      @output = {
-        file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
-        meta: {failures: 0, passes: 0, pending: 0},
-        examples: {}
-      }
-
-      to_h.values.each { |example| calculate(example) }
-
-      @output[:meta][:total_run_time] = @run_times.sum
-      @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
-      @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
-      @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
-
-      @output
-    end
-
-    private
-
-    def calculate(example)
-      @run_times << example[:run_time]
-      @output[:file_totals][example[:file_path]] += example[:run_time]
-
-      started_at = Time.parse(example[:started_at])
-      finished_at = Time.parse(example[:finished_at])
-
-      @first_started_at = started_at if started_at < @first_started_at
-      @last_finished_at = finished_at if finished_at > @last_finished_at
-
-      case example[:status]
-      when "passed"
-        @output[:meta][:passes] += 1
-      when "failed"
-        @output[:meta][:failures] += 1
-      when "pending"
-        @output[:meta][:pending] += 1
-      end
-
-      @output[:examples][example[:id]] = example
-    end
-  end
-end
+require "specwrk/store/base"
+require "specwrk/store/worker_store"
+require "specwrk/store/pending_store"
+require "specwrk/store/processing_store"
+require "specwrk/store/completed_store"
+require "specwrk/store/bucket_store"

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.16.3"
+  VERSION = "0.17.0"
 end

data/lib/specwrk/web/endpoints/base.rb

@@ -21,23 +21,15 @@ module Specwrk
 
           payload # parse the payload before any locking
 
-          before_lock
-
           worker.first_seen_at ||= Time.now
           worker.last_seen_at = Time.now
 
-          final_response = with_lock do
-            started_at = metadata[:started_at] ||= Time.now.iso8601
-            @started_at = Time.parse(started_at)
+          started_at = metadata[:started_at] ||= Time.now.iso8601
+          @started_at = Time.parse(started_at)
 
-            with_response
+          with_response.tap do |response|
+            response[1]["x-specwrk-status"] = worker_status.to_s
           end
-
-          after_lock
-
-          final_response[1]["x-specwrk-status"] = worker_status.to_s
-
-          final_response
         end
 
         def with_response
@@ -48,16 +40,6 @@ module Specwrk
 
         attr_reader :request
 
-        def skip_lock
-          false
-        end
-
-        def before_lock
-        end
-
-        def after_lock
-        end
-
         def not_found
           if request.head?
             [404, {}, []]
@@ -133,12 +115,10 @@ module Specwrk
         end
 
         def with_lock
-          if skip_lock
-            yield
-          else
-            with_mutex do
-              Store.with_lock(URI(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///")), run_id) { yield }
-            end
+          return yield unless run_id
+
+          with_mutex do
+            Store.with_lock(URI(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///")), run_id) { yield }
           end
         end
 

data/lib/specwrk/web/endpoints/complete_and_pop.rb

@@ -11,19 +11,20 @@ module Specwrk
         def with_response
           completed.merge!(completed_examples)
           processing.delete(*(completed_examples.keys + retry_examples.keys))
-          pending.merge!(retry_examples)
+
+          with_lock do
+            pending.merge!(retry_examples)
+          end
+
           failure_counts.merge!(retry_examples_new_failure_counts)
 
+          update_run_times
+
           with_pop_response
         end
 
         private
 
-        def before_lock
-          processing_examples
-          completed_examples
-        end
-
         def all_examples
           @all_examples ||= payload.map { |example| [example[:id], example] if processing_examples[example[:id]] }.compact.to_h
         end
@@ -71,9 +72,8 @@ module Specwrk
           @completed_examples_status_counts ||= completed_examples.values.map { |example| example[:status] }.tally
         end
 
-        def after_lock
-          # We don't care about exact values here, just approximate run times are fine
-          # So if we overwrite run times from another process it is nbd
+        def update_run_times
+          # Rough run time tracking does not require holding the store lock
           run_times.merge! run_time_data
 
           # workers are single process, single-threaded, so safe to do this work without the lock

data/lib/specwrk/web/endpoints/health.rb

@@ -9,12 +9,6 @@ module Specwrk
         def with_response
           ok
         end
-
-        private
-
-        def skip_lock
-          false
-        end
       end
     end
   end

data/lib/specwrk/web/endpoints/heartbeat.rb

@@ -9,12 +9,6 @@ module Specwrk
         def with_response
           ok
         end
-
-        private
-
-        def skip_lock
-          false
-        end
       end
     end
   end

data/lib/specwrk/web/endpoints/popable.rb

@@ -16,8 +16,9 @@ module Specwrk
           elsif completed.any? && processing.empty?
             [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           elsif expired_examples.length.positive?
-            pending.merge!(expired_examples.each { |_id, example| example[:worker_id] = worker_id })
-            processing.delete(*expired_examples.keys)
+            expired_examples.each { |_id, example| example[:worker_id] = worker_id }
+            with_lock { pending.push_examples(expired_examples.values) }
+            processing.delete(*expired_examples.keys.map(&:to_s))
             @examples = nil
 
             [200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
@@ -28,7 +29,11 @@ module Specwrk
 
         def examples
           @examples ||= begin
-            examples = pending.shift_bucket
+            bucket_id = with_lock { pending.shift_bucket }
+            return [] if bucket_id.nil?
+
+            bucket = pending.bucket_store_for(bucket_id)
+            examples = bucket.examples
 
             processing_data = examples.map do |example|
               [
@@ -37,13 +42,14 @@ module Specwrk
             end
 
             processing.merge!(processing_data.to_h)
+            bucket.clear
 
             examples
           end
         end
 
         def expired_examples
-          return unless processing.any?
+          return {} unless processing.any?
 
           @expired_examples ||= processing.to_h.select { |_id, example| expired?(example) }
         end

data/lib/specwrk/web/endpoints/report.rb

@@ -13,12 +13,6 @@ module Specwrk
 
           [200, {"content-type" => "application/json"}, [JSON.generate(completed_dump)]]
         end
-
-        private
-
-        def skip_lock
-          true
-        end
       end
     end
   end

data/lib/specwrk/web/endpoints/seed.rb

@@ -6,23 +6,22 @@ module Specwrk
   class Web
     module Endpoints
       class Seed < Base
-        def before_lock
+        def with_response
           examples_with_run_times
-        end
 
-        def with_response
           pending.clear
           processing.clear
           failure_counts.clear
+          completed.clear
 
           pending.max_retries = payload.fetch(:max_retries, "0").to_i
 
           new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
           pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
 
-          pending.merge!(examples_with_run_times)
-          processing.clear
-          completed.clear
+          with_lock do
+            pending.merge!(examples_with_run_times)
+          end
 
           ok
         end

data/lib/specwrk/web/endpoints/shutdown.rb

@@ -14,10 +14,6 @@ module Specwrk
 
         private
 
-        def skip_lock
-          true
-        end
-
         def interupt!
           Thread.new do
             # give the socket a moment to flush the response

data/lib/specwrk/worker.rb

@@ -54,6 +54,7 @@ module Specwrk
         end
       end
 
+      executor.flush_log
       executor.final_output.tap(&:rewind).each_line { |line| final_output.write line }
 
       @heartbeat_thread.kill

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.16.3
+  version: 0.17.0
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -209,9 +209,15 @@ files:
 - lib/specwrk/list_examples.rb
 - lib/specwrk/seed_loop.rb
 - lib/specwrk/store.rb
+- lib/specwrk/store/base.rb
 - lib/specwrk/store/base_adapter.rb
+- lib/specwrk/store/bucket_store.rb
+- lib/specwrk/store/completed_store.rb
 - lib/specwrk/store/file_adapter.rb
 - lib/specwrk/store/memory_adapter.rb
+- lib/specwrk/store/pending_store.rb
+- lib/specwrk/store/processing_store.rb
+- lib/specwrk/store/worker_store.rb
 - lib/specwrk/version.rb
 - lib/specwrk/watcher.rb
 - lib/specwrk/web.rb