specwrk 0.10.2 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4570e3aaeb01c00647b8671803076848bf28dee5068ec0cde66c88effea85bbe
-  data.tar.gz: 1d6709fc4b518ca5dcbad15e59820ba855adebb5dc6830ec26cc7f92d59f597d
+  metadata.gz: d642db71c542c4b2dbf17038db546a34f713ffccb010dc903e4e7a96c8071016
+  data.tar.gz: 00fd8258a2441acd31c4a484c2e039ad5e83028e8c10eef509cdca070daab1c8
 SHA512:
-  metadata.gz: 3868e891fc3c14a8ce281b747d8e5b0c271e55f946f1fe1d23aa4d88d940ee40ea966fbf32bce5711ca1597c7870a8a4b1aa1bac610f117a9695888015eee6db
-  data.tar.gz: 385faa8d85c751eca15e1203ab626ec8d9abe3584e79265550dc52b18c81327f6a837535827dd63d56bc5fb0671fe11c8bd91698a96dc0d9a82d36a23edc7430
+  metadata.gz: 6c88c00baa4d0455e8b928e276e6c66cf66fd881f7e9193a4e19f665a93d7b177b18e5f2cd00a8605ef4613e9607634791527898836f85f99bec0d3faf17e3d7
+  data.tar.gz: 7370d6c27e8b052a17bcc93d84fdbea18efab35792ec6f010cc21d05a45ed8aad4cc29e1c24e4b5e10aca65787f8fb08bc7bf94fa4e928293e41405a300259d6

data/README.md

@@ -51,14 +51,15 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN, default: "main"
   --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT, default: "5"
-  --id=VALUE                        # The identifier for this worker. Default specwrk-worker(-COUNT_INDEX), default: "specwrk-worker"
+  --id=VALUE                        # The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used
   --count=VALUE, -c VALUE           # The number of worker processes you want to start, default: 1
   --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
   --seed-waits=VALUE, -w VALUE      # Number of times the worker will wait for examples to be seeded to the server. 1sec between attempts. Overrides SPECWRK_SEED_WAITS, default: "10"
   --port=VALUE, -p VALUE            # Server port. Overrides SPECWRK_SRV_PORT, default: "5138"
   --bind=VALUE, -b VALUE            # Server bind address. Overrides SPECWRK_SRV_BIND, default: "127.0.0.1"
+  --store-uri=VALUE                 # Directory where server state is stored. Required for multi-node or multi-process servers.
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]verbose                    # Run in verbose mode. Default false., default: false
+  --[no-]verbose                    # Run in verbose mode, default: false
   --help, -h                        # Print this help
 ```
 
@@ -80,10 +81,11 @@ Options:
   --port=VALUE, -p VALUE            # Server port. Overrides SPECWRK_SRV_PORT, default: "5138"
   --bind=VALUE, -b VALUE            # Server bind address. Overrides SPECWRK_SRV_BIND, default: "127.0.0.1"
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
-  --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
+  --output=VALUE, -o VALUE          # Directory where worker or server output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
+  --store-uri=VALUE                 # Directory where server state is stored. Required for multi-node or multi-process servers.
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]verbose                    # Run in verbose mode. Default false., default: false
-  --[no-]single-run                 # Act on shutdown requests from clients. Default: false., default: false
+  --[no-]verbose                    # Run in verbose mode, default: false
+  --[no-]single-run                 # Act on shutdown requests from clients, default: false
   --help, -h                        # Print this help
 ```
 
@@ -196,7 +198,7 @@ Start a persistent Queue Server given one of the following methods
 
 ### Configuring your Queue Server
 - Secure your server with a key either with the `SPECWRK_SRV_KEY` environment variable or `--key` CLI option
-- Configure the server output to be a persisted volume so your timings survive between restarts with  the `SPECWRK_OUT` environment variable or `--out` CLI option 
+- Configure the server output to be a persisted volume so your timings survive between system restarts with the `SPECWRK_SRV_STORE_URI` environment variable or `--store-uri` CLI option. By default, `memory:///` will be used for the run's data stores (so run data will no survive server restarts) while `file://#{Dir.tmpdir}` will be used for run timings. Pass `--store-uri file:///whatever/absolute/path` to store all data on disk (required for multiple server processes).
 
 See [specwrk serve --help](#specwrk-serve) for all possible configuration options.
 

data/docker/pitchfork.conf

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-worker_processes 1
+worker_processes ENV.fetch("PITCHFORK_WORKERS", "1").to_i
 listen "localhost:3000", backlog: 2048
-timeout 301
+timeout ENV.fetch("IDLE_TIMEOUT", "300").to_i + 1

data/lib/specwrk/cli.rb

@@ -74,13 +74,15 @@ module Specwrk
         base.unique_option :port, type: :integer, default: ENV.fetch("SPECWRK_SRV_PORT", "5138"), aliases: ["-p"], desc: "Server port. Overrides SPECWRK_SRV_PORT"
         base.unique_option :bind, type: :string, default: ENV.fetch("SPECWRK_SRV_BIND", "127.0.0.1"), aliases: ["-b"], desc: "Server bind address. Overrides SPECWRK_SRV_BIND"
         base.unique_option :key, type: :string, aliases: ["-k"], default: ENV.fetch("SPECWRK_SRV_KEY", ""), desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
-        base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker output is stored. Overrides SPECWRK_OUT"
+        base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker or server output is stored. Overrides SPECWRK_OUT"
+        base.unique_option :store_uri, type: :string, desc: "Directory where server state is stored. Required for multi-node or multi-process servers."
         base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWERK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY"
-        base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode. Default false."
+        base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode"
       end
 
-      on_setup do |port:, bind:, output:, key:, group_by:, verbose:, **|
+      on_setup do |port:, bind:, output:, key:, group_by:, verbose:, **opts|
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
+        ENV["SPECWRK_SRV_STORE_URI"] = opts[:store_uri] if opts.key? :store_uri
         ENV["SPECWRK_SRV_VERBOSE"] = "1" if verbose
 
         ENV["SPECWRK_SRV_PORT"] = port
@@ -158,7 +160,7 @@ module Specwrk
       include Servable
 
       desc "Start a queue server"
-      option :single_run, type: :boolean, default: false, desc: "Act on shutdown requests from clients. Default: false."
+      option :single_run, type: :boolean, default: false, desc: "Act on shutdown requests from clients"
 
       def call(single_run:, **args)
         ENV["SPECWRK_SRV_SINGLE_RUN"] = "1" if single_run

data/lib/specwrk/cli_reporter.rb

@@ -16,8 +16,8 @@ module Specwrk
         return 1
       end
 
-      puts "\nFinished in #{total_duration} " \
-                          "(total execution time of #{total_run_time})\n"
+      puts "\nFinished in #{Specwrk.human_readable_duration total_duration} " \
+                          "(total execution time of #{Specwrk.human_readable_duration total_run_time})\n"
 
       client.shutdown
 

data/lib/specwrk/client.rb

@@ -44,7 +44,7 @@ module Specwrk
       raise Errno::ECONNREFUSED unless connected
     end
 
-    attr_reader :last_request_at
+    attr_reader :last_request_at, :retry_count
 
     def initialize
       @mutex = Mutex.new
@@ -120,6 +120,16 @@ module Specwrk
       else
         raise UnhandledResponseError.new("#{response.code}: #{response.body}")
       end
+    rescue Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout => e
+      @retry_count ||= 0
+      @retry_count += 1
+
+      raise e if @retry_count == 5
+
+      warn e
+      sleep @retry_count
+
+      retry
     end
 
     def seed(examples)
@@ -161,7 +171,7 @@ module Specwrk
     def make_request(request)
       @mutex.synchronize do
         @last_request_at = Time.now
-        @http.request(request)
+        @http.request(request).tap { @retry_count = 0 }
       end
     end
 

data/lib/specwrk/store/base_adapter.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "uri"
+
+require "specwrk/store"
+
+module Specwrk
+  class Store
+    class BaseAdapter
+      class << self
+        def with_lock(_uri, _key)
+          yield
+        end
+      end
+
+      def initialize(uri, scope)
+        @uri = uri
+        @scope = scope
+      end
+
+      def [](key)
+        raise "Not implemented"
+      end
+
+      def []=(key, value)
+        raise "Not implemented"
+      end
+
+      def keys
+        raise "Not implemented"
+      end
+
+      def clear
+        raise "Not implemented"
+      end
+
+      def delete(*keys)
+        raise "Not implemented"
+      end
+
+      def merge!(h2)
+        raise "Not implemented"
+      end
+
+      def multi_read(*read_keys)
+        raise "Not implemented"
+      end
+
+      def multi_write(hash)
+        raise "Not implemented"
+      end
+
+      def empty?
+        raise "Not implemented"
+      end
+
+      private
+
+      attr_reader :uri, :scope
+    end
+  end
+end

data/lib/specwrk/store/file_adapter.rb

@@ -3,17 +3,31 @@
 require "json"
 require "base64"
 
-require "specwrk/store"
+require "specwrk/store/base_adapter"
 
 module Specwrk
   class Store
-    class FileAdapter
+    class FileAdapter < BaseAdapter
       EXT = ".wrk.json"
 
       @work_queue = Queue.new
       @threads = []
 
       class << self
+        def with_lock(uri, key)
+          lock_file_path = File.join(uri.path, "#{key}.lock").tap do |path|
+            FileUtils.mkdir_p(uri.path)
+          end
+
+          lock_file = File.open(lock_file_path, "a")
+
+          Thread.pass until lock_file.flock(File::LOCK_EX)
+
+          yield
+        ensure
+          lock_file.flock(File::LOCK_UN)
+        end
+
         def schedule_work(&blk)
           start_threads!
           @work_queue.push blk
@@ -32,11 +46,6 @@ module Specwrk
         end
       end
 
-      def initialize(path)
-        @path = path
-        FileUtils.mkdir_p(@path)
-      end
-
       def [](key)
         content = read(key.to_s)
         return unless content
@@ -51,7 +60,6 @@ module Specwrk
         else
           filename = filename_for_key(key_string)
           write(filename, JSON.generate(value))
-          known_key_pairs[key_string] = filename
         end
       end
 
@@ -60,18 +68,14 @@ module Specwrk
       end
 
       def clear
-        FileUtils.rm_rf(@path)
-        FileUtils.mkdir_p(@path)
-
-        @known_key_pairs = nil
+        FileUtils.rm_rf(path)
+        FileUtils.mkdir_p(path)
       end
 
       def delete(*keys)
-        filenames = keys.map { |key| known_key_pairs[key] }.compact
+        filenames = keys.map { |key| filename_for_key key }.compact
 
         FileUtils.rm_f(filenames)
-
-        keys.each { |key| known_key_pairs.delete(key) }
       end
 
       def merge!(h2)
@@ -79,8 +83,6 @@ module Specwrk
       end
 
       def multi_read(*read_keys)
-        known_key_pairs # precache before each thread tries to look them up
-
         result_queue = Queue.new
 
         read_keys.each do |key|
@@ -103,8 +105,6 @@ module Specwrk
       end
 
       def multi_write(hash)
-        known_key_pairs # precache before each thread tries to look them up
-
         result_queue = Queue.new
 
         hash_with_filenames = hash.map { |key, value| [key.to_s, [filename_for_key(key.to_s), value]] }.to_h
@@ -117,11 +117,10 @@ module Specwrk
         end
 
         Thread.pass until result_queue.length == hash.length
-        hash_with_filenames.each { |key, (filename, _value)| known_key_pairs[key] = filename }
       end
 
       def empty?
-        Dir.empty? @path
+        Dir.empty? path
       end
 
       private
@@ -136,27 +135,23 @@ module Specwrk
       end
 
       def read(key)
-        File.read(known_key_pairs[key]) if known_key_pairs.key? key
+        filename = filename_for_key key
+        File.read(filename)
+      rescue Errno::ENOENT
+        nil
       end
 
       def filename_for_key(key)
         File.join(
-          @path,
-          [
-            counter_prefix(key),
-            encode_key(key)
-          ].join("_")
+          path,
+          encode_key(key)
         ) + EXT
       end
 
-      def counter_prefix(key)
-        count = keys.index(key) || counter.tap { @counter += 1 }
-
-        "%012d" % count
-      end
-
-      def counter
-        @counter ||= keys.length
+      def path
+        @path ||= File.join(uri.path, scope).tap do |full_path|
+          FileUtils.mkdir_p(full_path)
+        end
       end
 
       def encode_key(key)
@@ -164,18 +159,18 @@ module Specwrk
       end
 
       def decode_key(key)
-        encoded_key_part = File.basename(key).delete_suffix(EXT).split(/\A\d+_/).last
+        encoded_key_part = File.basename(key).delete_suffix(EXT)
         padding_count = (4 - encoded_key_part.length % 4) % 4
 
         Base64.urlsafe_decode64(encoded_key_part + ("=" * padding_count))
       end
 
       def known_key_pairs
-        @known_key_pairs ||= Dir.entries(@path).sort.map do |filename|
+        Dir.entries(path).sort.map do |filename|
           next if filename.start_with? "."
           next unless filename.end_with? EXT
 
-          file_path = File.join(@path, filename)
+          file_path = File.join(path, filename)
           [decode_key(filename), file_path]
         end.compact.to_h
       end

data/lib/specwrk/store/memory_adapter.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "specwrk/store/base_adapter"
+
+module Specwrk
+  class Store
+    class MemoryAdapter < BaseAdapter
+      @@stores = Hash.new { |hash, key| hash[key] = {} }
+
+      class << self
+        def clear
+          @@stores.values.each(&:clear)
+        end
+      end
+
+      def [](key)
+        store[key]
+      end
+
+      def []=(key, value)
+        store[key] = value
+      end
+
+      def keys
+        store.keys
+      end
+
+      def clear
+        store.clear
+      end
+
+      def delete(*keys)
+        keys.each { |key| store.delete(key) }
+      end
+
+      def merge!(h2)
+        store.merge!(h2)
+      end
+
+      def multi_read(*read_keys)
+        store.slice(*read_keys)
+      end
+
+      def multi_write(hash)
+        merge!(hash)
+      end
+
+      def empty?
+        store.keys.length.zero?
+      end
+
+      private
+
+      def store
+        @store ||= @@stores[scope]
+      end
+    end
+  end
+end

data/lib/specwrk/store.rb

@@ -1,14 +1,31 @@
 # frozen_string_literal: true
 
 require "time"
-require "json"
-
-require "specwrk/store/file_adapter"
 
 module Specwrk
   class Store
-    def initialize(path)
-      @path = path
+    class << self
+      def with_lock(uri, key)
+        adapter_klass(uri).with_lock(uri, key) { yield }
+      end
+
+      def adapter_klass(uri)
+        case uri.scheme
+        when "memory"
+          require "specwrk/store/memory_adapter" unless defined?(MemoryAdapter)
+
+          MemoryAdapter
+        when "file"
+          require "specwrk/store/file_adapter" unless defined?(FileAdapter)
+
+          FileAdapter
+        end
+      end
+    end
+
+    def initialize(uri_string, scope)
+      @uri = URI(uri_string)
+      @scope = scope
     end
 
     def [](key)
@@ -72,17 +89,16 @@ module Specwrk
 
     private
 
-    def adapter
-      @adapter ||= FileAdapter.new(@path)
-    end
+    attr_reader :uri, :scope
 
-    def mutex
-      @mutex ||= self.class.mutex_for(@path)
+    def adapter
+      @adapter ||= self.class.adapter_klass(uri).new uri, scope
     end
   end
 
   class PendingStore < Store
     RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
+    ORDER_KEY = :____order
 
     def run_time_bucket_maximum=(val)
       @run_time_bucket_maximum = self[RUN_TIME_BUCKET_MAXIMUM_KEY] = val
@@ -92,6 +108,41 @@ module Specwrk
       @run_time_bucket_maximum ||= self[RUN_TIME_BUCKET_MAXIMUM_KEY]
     end
 
+    def order=(val)
+      @order = nil
+
+      self[ORDER_KEY] = if val.nil? || val.length.zero?
+        nil
+      else
+        val
+      end
+    end
+
+    def order
+      @order ||= self[ORDER_KEY] || []
+    end
+
+    def keys
+      return super if order.length.zero?
+
+      order
+    end
+
+    def merge!(hash)
+      super
+      self.order = hash.keys
+    end
+
+    def clear
+      @order = nil
+      super
+    end
+
+    def reload
+      @order = nil
+      super
+    end
+
     def shift_bucket
       return bucket_by_file unless run_time_bucket_maximum&.positive?
 
@@ -131,6 +182,7 @@ module Specwrk
       end
 
       delete(*consumed_keys)
+      self.order = order - consumed_keys
       bucket
     end
 
@@ -142,7 +194,7 @@ module Specwrk
       estimated_run_time_total = 0
 
       catch(:full) do
-        keys.each_slice(25).each do |key_group|
+        keys.each_slice(24).each do |key_group|
           examples = multi_read(*key_group)
 
           examples.each do |key, example|
@@ -156,10 +208,30 @@ module Specwrk
       end
 
       delete(*consumed_keys)
+      self.order = order - consumed_keys
       bucket
     end
   end
 
+  class ProcessingStore < Store
+    def expired
+      @expired ||= begin
+        bucket = []
+
+        keys.each_slice(24).each do |key_group|
+          examples = multi_read(*key_group)
+          examples.each do |id, example|
+            next if example[:completion_threshold].nil?
+
+            bucket << [id, example] if example[:completion_threshold] < Time.now.to_i
+          end
+        end
+
+        bucket.to_h
+      end
+    end
+  end
+
   class CompletedStore < Store
     def dump
       @run_times = []

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.10.2"
+  VERSION = "0.12.0"
 end

data/lib/specwrk/web/auth.rb

@@ -11,7 +11,7 @@ module Specwrk
       end
 
       def call(env)
-        env[:request] ||= Rack::Request.new(env)
+        @request = env[:request] ||= Rack::Request.new(env)
 
         return @app.call(env) if [nil, ""].include? ENV["SPECWRK_SRV_KEY"]
         return @app.call(env) if @excluded_paths.include? env[:request].path_info
@@ -28,7 +28,11 @@ module Specwrk
       private
 
       def unauthorized
-        [401, {"content-type" => "application/json"}, ["Unauthorized"]]
+        if @request.head?
+          [401, {}, []]
+        else
+          [401, {"content-type" => "application/json"}, ["Unauthorized"]]
+        end
       end
     end
   end

data/lib/specwrk/web/endpoints.rb

@@ -51,11 +51,19 @@ module Specwrk
         end
 
         def not_found
-          [404, {"content-type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
+          if request.head?
+            [404, {}, []]
+          else
+            [404, {"content-type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
+          end
         end
 
         def ok
-          [200, {"content-type" => "text/plain"}, ["OK, 'ol chap"]]
+          if request.head?
+            [200, {}, []]
+          else
+            [200, {"content-type" => "text/plain"}, ["OK, 'ol chap"]]
+          end
         end
 
         def payload
@@ -71,52 +79,35 @@ module Specwrk
         end
 
         def pending
-          @pending ||= PendingStore.new(File.join(datastore_path, "pending"))
+          @pending ||= PendingStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "pending"))
         end
 
         def processing
-          @processing ||= Store.new(File.join(datastore_path, "processing"))
+          @processing ||= ProcessingStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "processing"))
         end
 
         def completed
-          @completed ||= CompletedStore.new(File.join(datastore_path, "completed"))
+          @completed ||= CompletedStore.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "completed"))
         end
 
         def metadata
-          @metadata ||= Store.new(File.join(datastore_path, "metadata"))
+          @metadata ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "metadata"))
         end
 
         def run_times
-          @run_times ||= Store.new(File.join(ENV["SPECWRK_OUT"], "run_times"))
+          @run_times ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "file://#{File.join(Dir.tmpdir, "specwrk")}"), "run_times")
         end
 
         def worker
-          @worker ||= Store.new(File.join(datastore_path, "workers", request.get_header("HTTP_X_SPECWRK_ID").to_s))
+          @worker ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "workers", request.get_header("HTTP_X_SPECWRK_ID").to_s))
         end
 
         def run_id
           request.get_header("HTTP_X_SPECWRK_RUN")
         end
 
-        def run_report_file_path
-          @run_report_file_path ||= File.join(datastore_path, "#{started_at.strftime("%Y%m%dT%H%M%S")}-report.json").to_s
-        end
-
-        def datastore_path
-          @datastore_path ||= File.join(ENV["SPECWRK_OUT"], run_id).to_s.tap do |path|
-            FileUtils.mkdir_p(path) unless File.directory?(path)
-          end
-        end
-
         def with_lock
-          Thread.pass until lock_file.flock(File::LOCK_EX)
-          yield
-        ensure
-          lock_file.flock(File::LOCK_UN)
-        end
-
-        def lock_file
-          @lock_file ||= File.open(File.join(datastore_path, "lock"), "a")
+          Store.with_lock(URI(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///")), "server") { yield }
         end
       end
 
@@ -125,7 +116,7 @@ module Specwrk
 
       class Health < Base
         def with_response
-          [200, {}, []]
+          ok
         end
       end
 
@@ -222,54 +213,63 @@ module Specwrk
         end
       end
 
-      class Pop < Base
-        def with_response
-          @examples = pending.shift_bucket
-
-          processing_data = @examples.map { |example| [example[:id], example] }.to_h
-          processing.merge!(processing_data)
+      class Popable < Base
+        private
 
-          if @examples.any?
-            [200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
+        def with_pop_response
+          if examples.any?
+            [200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
           elsif pending.empty? && processing.empty? && completed.empty?
             [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
           elsif completed.any? && processing.empty?
             [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
+          elsif processing.any? && processing.expired.keys.any?
+            pending.merge!(processing.expired)
+            processing.delete(*processing.expired.keys)
+            @examples = nil
+
+            [200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
           else
             not_found
           end
         end
-      end
 
-      class CompleteAndPop < Base
-        def with_response
-          completed.merge!(completed_examples)
-          run_times.merge! run_time_data
-          processing.delete(*completed_examples.keys)
+        def examples
+          @examples ||= begin
+            examples = pending.shift_bucket
+            maximum_completion_threshold = (Time.now + ((pending.run_time_bucket_maximum || 30) * 2)).to_i
 
-          @examples = pending.shift_bucket
+            processing_data = examples.map do |example|
+              example_run_time_completion_threshold = (Time.now + example[:expected_run_time].to_f * 2).to_i
 
-          processing_data = @examples.map { |example| [example[:id], example] }.to_h
-          processing.merge!(processing_data)
+              [
+                example[:id], example.merge(completion_threshold: [maximum_completion_threshold, example_run_time_completion_threshold].compact.max)
+              ]
+            end
 
-          if @examples.any?
-            [200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
-          elsif pending.empty? && processing.empty? && completed.empty?
-            [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
-          elsif completed.any? && processing.empty?
-            [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
-          else
-            not_found
+            processing.merge!(processing_data.to_h)
+
+            examples
           end
         end
+      end
 
-        private
+      class Pop < Popable
+        def with_response
+          with_pop_response
+        end
+      end
 
-        def before_lock
-          completed_examples
-          run_time_data
+      class CompleteAndPop < Popable
+        def with_response
+          completed.merge!(completed_examples)
+          processing.delete(*completed_examples.keys)
+
+          with_pop_response
         end
 
+        private
+
         def completed_examples
           @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
         end
@@ -277,8 +277,7 @@ module Specwrk
         # We don't care about exact values here, just approximate run times are fine
         # So if we overwrite run times from another process it is nbd
         def after_lock
-          # run_time_data = payload.map { |example| [example[:id], example[:run_time]] }.to_h
-          # run_times.merge! run_time_data
+          run_times.merge! run_time_data
         end
 
         def run_time_data

data/lib/specwrk/worker/executor.rb

@@ -40,6 +40,7 @@ module Specwrk
         completion_formatter.examples.clear
 
         RSpec.clear_examples
+        RSpec.configuration.backtrace_formatter.filter_gem "specwrk"
 
         # see https://github.com/rspec/rspec-core/pull/2723
         if Gem::Version.new(RSpec::Core::Version::STRING) <= Gem::Version.new("3.9.1")

data/lib/specwrk.rb

@@ -38,5 +38,21 @@ module Specwrk
 
       exited_pids
     end
+
+    def human_readable_duration(total_seconds, precision: 2)
+      secs = total_seconds.to_f
+      hours = (secs / 3600).to_i
+      mins = ((secs % 3600) / 60).to_i
+      seconds = secs % 60
+
+      parts = []
+      parts << "#{hours}h" if hours.positive?
+      parts << "#{mins}m" if mins.positive?
+      if seconds.positive?
+        sec_str = format("%0.#{precision}f", seconds).sub(/\.?0+$/, "")
+        parts << "#{sec_str}s"
+      end
+      parts.join(" ")
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.12.0
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -192,7 +192,9 @@ files:
 - lib/specwrk/hookable.rb
 - lib/specwrk/list_examples.rb
 - lib/specwrk/store.rb
+- lib/specwrk/store/base_adapter.rb
 - lib/specwrk/store/file_adapter.rb
+- lib/specwrk/store/memory_adapter.rb
 - lib/specwrk/version.rb
 - lib/specwrk/web.rb
 - lib/specwrk/web/app.rb