specwrk 0.8.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2122e3e3925fdf1c12b6d33899c52ba1b8287e46e7518754ae0ca8188ac149ce
-  data.tar.gz: 89579242f18a5f17aac60ac52664e1197cf61ef35228d142298b4ad250057a25
+  metadata.gz: 656884cedc1e6851a42054d696fb0c8cbb3c7afffa48935340c973d824cc6cc0
+  data.tar.gz: e8b562b8070e5342d6343b299a80fe0bc98a17060632aa369f5ded91ffecb820
 SHA512:
-  metadata.gz: c357e649df3ad4533290c7e53be49d1f4a3bf86870b5cc01ed4bc2219393bc026d4056e65c82b21c9f251724fe3adcebf6efda7a2643b85d873e4ede7efe17d3
-  data.tar.gz: 1aea05a1f33cd4bfeddbbe76d7d97c7dd3535ef4d1c24b5116042bcfe37201acc7a70c0839a6ef28ba9a5ecd9380535a7007c8a118e5277fc8d798899d7f28e1
+  metadata.gz: a536a0bf7de4959ca820afa5e9249024181f198d073ca648f040f8f4276f63572ac74a1cdff481b9e050b1983f7af7eac5b2d31c3271dff911ae88aa75092f27
+  data.tar.gz: eddaa254e6254727f950006b30b653402a33acd827b1060baf583840d99ae2d41dec0ab4a15c11754c03db3f2664ea348efefc1b32b95b494c56287a0d3b88d0

data/README.md

@@ -58,7 +58,6 @@ Options:
   --port=VALUE, -p VALUE            # Server port. Overrides SPECWRK_SRV_PORT, default: "5138"
   --bind=VALUE, -b VALUE            # Server bind address. Overrides SPECWRK_SRV_BIND, default: "127.0.0.1"
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]single-seed-per-run        # Only allow one seed per run. Useful for CI where many nodes may seed at the same time, default: false
   --[no-]verbose                    # Run in verbose mode. Default false., default: false
   --help, -h                        # Print this help
 ```
@@ -83,7 +82,6 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]single-seed-per-run        # Only allow one seed per run. Useful for CI where many nodes may seed at the same time, default: false
   --[no-]verbose                    # Run in verbose mode. Default false., default: false
   --[no-]single-run                 # Act on shutdown requests from clients. Default: false., default: false
   --help, -h                        # Print this help

data/docker/Dockerfile.server

@@ -1,6 +1,11 @@
 FROM ruby:3.4-alpine
 
-RUN apk add --no-cache build-base 
+RUN apk add --no-cache \
+      build-base \
+      ruby-dev \
+      linux-headers \
+      zlib-dev \
+      libffi-dev
 
 WORKDIR /app
 
@@ -8,15 +13,14 @@ RUN mkdir .specwrk/
 
 ARG SPECWRK_SRV_PORT=5138
 ARG SPECWRK_VERSION=latest
-ARG GEM_FILE=specwrk-$SPECWRK_VERSION.gem
+ARG GEMFILE=specwrk-$SPECWRK_VERSION.gem
 
-COPY $GEM_FILE ./
-RUN gem install ./$GEM_FILE --no-document
-RUN rm ./$GEM_FILE
+COPY $GEMFILE ./
+RUN gem install ./$GEMFILE --no-document
+RUN rm ./$GEMFILE
 
-RUN gem install pitchfork thruster
+RUN gem install puma thruster
 COPY config.ru ./
-COPY docker/pitchfork.conf ./
 
 COPY docker/entrypoint.server.sh /usr/local/bin/entrypoint
 RUN chmod +x /usr/local/bin/entrypoint

data/docker/entrypoint.server.sh

@@ -2,6 +2,6 @@
 
 export THRUSTER_HTTP_PORT=${PORT:-5138}
 export THRUSTER_TARGET_PORT=3000
-export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-300}
+export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-305}
 
-exec thrust pitchfork -c pitchfork.conf
+exec thrust puma --workers 0 --bind tcp://127.0.0.1:3000 --threads ${PUMA_THREADS:-1} --workers ${PUMA_WORKERS:-0}

data/lib/specwrk/cli.rb

@@ -76,7 +76,6 @@ module Specwrk
         base.unique_option :key, type: :string, aliases: ["-k"], default: ENV.fetch("SPECWRK_SRV_KEY", ""), desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
         base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker output is stored. Overrides SPECWRK_OUT"
         base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWERK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY"
-        base.unique_option :single_seed_per_run, type: :boolean, default: false, desc: "Only allow one seed per run. Useful for CI where many nodes may seed at the same time"
         base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode. Default false."
       end
 
@@ -211,7 +210,8 @@ module Specwrk
           status "Server responding ✓"
           status "Seeding #{examples.length} examples..."
           Client.new.seed(examples)
-          status "Samples seeded ✓"
+          file_count = examples.group_by { |e| e[:file_path] }.keys.size
+          status "🌱 Seeded #{examples.size} examples across #{file_count} files"
         end
 
         if Specwrk.wait_for_pids_exit([seed_pid]).value?(1)

data/lib/specwrk/client.rb

@@ -105,6 +105,23 @@ module Specwrk
       (response.code == "200") ? true : raise(UnhandledResponseError.new("#{response.code}: #{response.body}"))
     end
 
+    def complete_and_fetch_examples(examples)
+      response = post "/complete_and_pop", body: examples.to_json
+
+      case response.code
+      when "200"
+        JSON.parse(response.body, symbolize_names: true)
+      when "204"
+        raise WaitingForSeedError
+      when "404"
+        raise NoMoreExamplesError
+      when "410"
+        raise CompletedAllExamplesError
+      else
+        raise UnhandledResponseError.new("#{response.code}: #{response.body}")
+      end
+    end
+
     def seed(examples)
       response = post "/seed", body: examples.to_json
 

data/lib/specwrk/store/file_adapter.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "json"
+require "base64"
+
+require "specwrk/store"
+
+module Specwrk
+  class Store
+    class FileAdapter
+      EXT = ".wrk.json"
+
+      THREAD_POOL = Class.new do
+        @work_queue = Queue.new
+
+        @threads = Array.new(ENV.fetch("SPECWRK_SRV_FILE_ADAPTER_THREAD_COUNT", "4").to_i) do
+          Thread.new do
+            loop do
+              @work_queue.pop.call
+            end
+          end
+        end
+
+        class << self
+          def schedule(&blk)
+            @work_queue.push blk
+          end
+        end
+      end
+
+      def initialize(path)
+        @path = path
+        FileUtils.mkdir_p(@path)
+      end
+
+      def [](key)
+        content = read(key.to_s)
+        return unless content
+
+        JSON.parse(content, symbolize_names: true)
+      end
+
+      def []=(key, value)
+        key_string = key.to_s
+        if value.nil?
+          delete(key_string)
+        else
+          filename = filename_for_key(key_string)
+          write(filename, JSON.generate(value))
+          known_key_pairs[key_string] = filename
+        end
+      end
+
+      def keys
+        known_key_pairs.keys
+      end
+
+      def clear
+        FileUtils.rm_rf(@path)
+        FileUtils.mkdir_p(@path)
+
+        @known_key_pairs = nil
+      end
+
+      def delete(*keys)
+        filenames = keys.map { |key| known_key_pairs[key] }.compact
+
+        FileUtils.rm_f(filenames)
+
+        keys.each { |key| known_key_pairs.delete(key) }
+      end
+
+      def merge!(h2)
+        multi_write(h2)
+      end
+
+      def multi_read(*read_keys)
+        known_key_pairs # precache before each thread tries to look them up
+
+        result_queue = Queue.new
+
+        read_keys.each do |key|
+          THREAD_POOL.schedule do
+            result_queue.push([key.to_s, read(key)])
+          end
+        end
+
+        Thread.pass until result_queue.length == read_keys.length
+
+        results = {}
+        until result_queue.empty?
+          result = result_queue.pop
+          next if result.last.nil?
+
+          results[result.first] = JSON.parse(result.last, symbolize_names: true)
+        end
+
+        read_keys.map { |key| [key.to_s, results[key.to_s]] if results.key?(key.to_s) }.compact.to_h # respect order requested in the returned hash
+      end
+
+      def multi_write(hash)
+        known_key_pairs # precache before each thread tries to look them up
+
+        result_queue = Queue.new
+
+        hash_with_filenames = hash.map { |key, value| [key.to_s, [filename_for_key(key.to_s), value]] }.to_h
+        hash_with_filenames.each do |key, (filename, value)|
+          content = JSON.generate(value)
+
+          THREAD_POOL.schedule do
+            result_queue << write(filename, content)
+          end
+        end
+
+        Thread.pass until result_queue.length == hash.length
+        hash_with_filenames.each { |key, (filename, _value)| known_key_pairs[key] = filename }
+      end
+
+      def empty?
+        Dir.empty? @path
+      end
+
+      private
+
+      def write(filename, content)
+        tmp_filename = [filename, "tmp"].join(".")
+
+        File.binwrite(tmp_filename, content)
+
+        FileUtils.mv tmp_filename, filename
+        true
+      end
+
+      def read(key)
+        File.read(known_key_pairs[key]) if known_key_pairs.key? key
+      end
+
+      def filename_for_key(key)
+        File.join(
+          @path,
+          [
+            counter_prefix(key),
+            encode_key(key)
+          ].join("_")
+        ) + EXT
+      end
+
+      def counter_prefix(key)
+        count = keys.index(key) || counter.tap { @counter += 1 }
+
+        "%012d" % count
+      end
+
+      def counter
+        @counter ||= keys.length
+      end
+
+      def encode_key(key)
+        Base64.urlsafe_encode64(key).delete("=")
+      end
+
+      def decode_key(key)
+        encoded_key_part = File.basename(key).delete_suffix(EXT).split(/\A\d+_/).last
+        padding_count = (4 - encoded_key_part.length % 4) % 4
+
+        Base64.urlsafe_decode64(encoded_key_part + ("=" * padding_count))
+      end
+
+      def known_key_pairs
+        @known_key_pairs ||= Dir.entries(@path).sort.map do |filename|
+          next if filename.start_with? "."
+          next unless filename.end_with? EXT
+
+          file_path = File.join(@path, filename)
+          [decode_key(filename), file_path]
+        end.compact.to_h
+      end
+    end
+  end
+end

data/lib/specwrk/store.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require "time"
+require "json"
+
+require "specwrk/store/file_adapter"
+
+module Specwrk
+  class Store
+    MUTEXES = {}
+    MUTEXES_MUTEX = Mutex.new # 🐢🐢🐢🐢
+
+    class << self
+      def mutex_for(path)
+        MUTEXES_MUTEX.synchronize do
+          MUTEXES[path] ||= Mutex.new
+        end
+      end
+    end
+
+    def initialize(path, thread_safe_reads: true)
+      @path = path
+      @thread_safe_reads = thread_safe_reads
+    end
+
+    def [](key)
+      sync(thread_safe: thread_safe_reads) { adapter[key.to_s] }
+    end
+
+    def multi_read(*keys)
+      sync(thread_safe: thread_safe_reads) { adapter.multi_read(*keys) }
+    end
+
+    def []=(key, value)
+      sync do
+        adapter[key.to_s] = value
+      end
+    end
+
+    def keys
+      all_keys = sync(thread_safe: thread_safe_reads) do
+        adapter.keys
+      end
+
+      all_keys.reject { |k| k.start_with? "____" }
+    end
+
+    def length
+      keys.length
+    end
+
+    def any?
+      !empty?
+    end
+
+    def empty?
+      sync(thread_safe: thread_safe_reads) do
+        adapter.empty?
+      end
+    end
+
+    def delete(*keys)
+      sync { adapter.delete(*keys) }
+    end
+
+    def merge!(h2)
+      h2.transform_keys!(&:to_s)
+      sync { adapter.merge!(h2) }
+    end
+
+    def clear
+      sync { adapter.clear }
+    end
+
+    def to_h
+      sync(thread_safe: thread_safe_reads) do
+        adapter.multi_read(*keys).transform_keys!(&:to_sym)
+      end
+    end
+
+    def inspect
+      reload.to_h.dup
+    end
+
+    # Bypass any cached values. Helpful when you have two instances
+    # of the same store where one mutates data and the other needs to check
+    # on the status of that data (i.e. endpoint tests)
+    def reload
+      @adapter = nil
+      self
+    end
+
+    private
+
+    attr_reader :thread_safe_reads
+
+    def sync(thread_safe: true)
+      if !thread_safe || mutex.owned?
+        yield
+      else
+        mutex.synchronize { yield }
+      end
+    end
+
+    def adapter
+      @adapter ||= FileAdapter.new(@path)
+    end
+
+    def mutex
+      @mutex ||= self.class.mutex_for(@path)
+    end
+  end
+
+  class PendingStore < Store
+    RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
+
+    def run_time_bucket_maximum=(val)
+      @run_time_bucket_maximum = self[RUN_TIME_BUCKET_MAXIMUM_KEY] = val
+    end
+
+    def run_time_bucket_maximum
+      @run_time_bucket_maximum ||= self[RUN_TIME_BUCKET_MAXIMUM_KEY]
+    end
+
+    def shift_bucket
+      sync do
+        return bucket_by_file unless run_time_bucket_maximum&.positive?
+
+        case ENV["SPECWRK_SRV_GROUP_BY"]
+        when "file"
+          bucket_by_file
+        else
+          bucket_by_timings
+        end
+      end
+    end
+
+    private
+
+    # Take elements from the hash where the file_path is the same
+    # Expects that the examples were merged in order of filename
+    def bucket_by_file
+      bucket = []
+      consumed_keys = []
+
+      all_keys = keys
+      key = all_keys.first
+      return [] if key.nil?
+
+      file_path = self[key][:file_path]
+
+      catch(:full) do
+        all_keys.each_slice(24).each do |key_group|
+          examples = multi_read(*key_group)
+
+          examples.each do |key, example|
+            throw :full unless example[:file_path] == file_path
+
+            bucket << example
+            consumed_keys << key
+          end
+        end
+      end
+
+      delete(*consumed_keys)
+      bucket
+    end
+
+    # Take elements from the hash until the average runtime bucket has filled
+    def bucket_by_timings
+      bucket = []
+      consumed_keys = []
+
+      estimated_run_time_total = 0
+
+      catch(:full) do
+        keys.each_slice(25).each do |key_group|
+          examples = multi_read(*key_group)
+
+          examples.each do |key, example|
+            estimated_run_time_total += example[:expected_run_time] || run_time_bucket_maximum
+            throw :full if estimated_run_time_total > run_time_bucket_maximum && bucket.length.positive?
+
+            bucket << example
+            consumed_keys << key
+          end
+        end
+      end
+
+      delete(*consumed_keys)
+      bucket
+    end
+  end
+
+  class CompletedStore < Store
+    def dump
+      @run_times = []
+      @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
+      @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
+
+      @output = {
+        file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
+        meta: {failures: 0, passes: 0, pending: 0},
+        examples: {}
+      }
+
+      to_h.values.each { |example| calculate(example) }
+
+      @output[:meta][:total_run_time] = @run_times.sum
+      @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
+      @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
+      @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
+
+      @output
+    end
+
+    private
+
+    def calculate(example)
+      @run_times << example[:run_time]
+      @output[:file_totals][example[:file_path]] += example[:run_time]
+
+      started_at = Time.parse(example[:started_at])
+      finished_at = Time.parse(example[:finished_at])
+
+      @first_started_at = started_at if started_at < @first_started_at
+      @last_finished_at = finished_at if finished_at > @last_finished_at
+
+      case example[:status]
+      when "passed"
+        @output[:meta][:passes] += 1
+      when "failed"
+        @output[:meta][:failures] += 1
+      when "pending"
+        @output[:meta][:pending] += 1
+      end
+
+      @output[:examples][example[:id]] = example
+    end
+  end
+end

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.8.0"
+  VERSION = "0.10.0"
 end

data/lib/specwrk/web/app.rb

@@ -21,8 +21,6 @@ require "specwrk/web/endpoints"
 module Specwrk
   class Web
     class App
-      REAP_INTERVAL = 330 # HTTP connection timeout + some buffer
-
       class << self
         def run!
           Process.setproctitle "specwrk-server"
@@ -74,10 +72,6 @@ module Specwrk
         end
       end
 
-      def initialize
-        @reaper_thread = Thread.new { reaper } unless ENV["SPECWRK_SRV_SINGLE_RUN"]
-      end
-
       def call(env)
         env[:request] ||= Rack::Request.new(env)
 
@@ -96,6 +90,8 @@ module Specwrk
           Endpoints::Pop
         when ["POST", "/complete"]
           Endpoints::Complete
+        when ["POST", "/complete_and_pop"]
+          Endpoints::CompleteAndPop
         when ["POST", "/seed"]
           Endpoints::Seed
         when ["GET", "/report"]
@@ -106,26 +102,6 @@ module Specwrk
           Endpoints::NotFound
         end
       end
-
-      def reaper
-        until Specwrk.force_quit
-          sleep REAP_INTERVAL
-
-          reap
-        end
-      end
-
-      def reap
-        Web::WORKERS.each do |run, workers|
-          most_recent_last_seen_at = workers.map { |id, worker| worker[:last_seen_at] }.max
-          next unless most_recent_last_seen_at
-
-          # Don't consider runs which aren't at least REAP_INTERVAL sec stale
-          if most_recent_last_seen_at < Time.now - REAP_INTERVAL
-            Web.clear_run_queues(run)
-          end
-        end
-      end
     end
   end
 end

data/lib/specwrk/web/auth.rb

@@ -28,7 +28,7 @@ module Specwrk
       private
 
       def unauthorized
-        [401, {"content-type" => "application/json"}, ["Unauthorized"]]
+        [401, {"Content-Type" => "application/json"}, ["Unauthorized"]]
       end
     end
   end

data/lib/specwrk/web/endpoints.rb

@@ -2,18 +2,41 @@
 
 require "json"
 
+require "specwrk/store"
+
 module Specwrk
   class Web
     module Endpoints
       class Base
+        attr_reader :started_at
+
         def initialize(request)
           @request = request
-
-          worker[:first_seen_at] ||= Time.now
-          worker[:last_seen_at] = Time.now
         end
 
         def response
+          before_lock
+
+          return with_response unless run_id # No run_id, no datastore usage in the endpoint
+
+          payload # parse the payload before any locking
+
+          worker[:first_seen_at] ||= Time.now.iso8601
+          worker[:last_seen_at] = Time.now.iso8601
+
+          final_response = with_lock do
+            started_at = metadata[:started_at] ||= Time.now.iso8601
+            @started_at = Time.parse(started_at)
+
+            with_response
+          end
+
+          after_lock
+
+          final_response
+        end
+
+        def with_response
           not_found
         end
 
@@ -21,15 +44,25 @@ module Specwrk
 
         attr_reader :request
 
+        def before_lock
+        end
+
+        def after_lock
+        end
+
         def not_found
-          [404, {"content-type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
+          [404, {"Content-Type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
         end
 
         def ok
-          [200, {"content-type" => "text/plain"}, ["OK, 'ol chap"]]
+          [200, {"Content-Type" => "text/plain"}, ["OK, 'ol chap"]]
         end
 
         def payload
+          return unless request.content_type&.start_with?("application/json")
+          return unless request.post? || request.put? || request.delete?
+          return if body.empty?
+
           @payload ||= JSON.parse(body, symbolize_names: true)
         end
 
@@ -37,24 +70,28 @@ module Specwrk
           @body ||= request.body.read
         end
 
-        def pending_queue
-          Web::PENDING_QUEUES[run_id]
+        def pending
+          @pending ||= PendingStore.new(File.join(datastore_path, "pending"))
+        end
+
+        def processing
+          @processing ||= Store.new(File.join(datastore_path, "processing"))
         end
 
-        def processing_queue
-          Web::PROCESSING_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
+        def completed
+          @completed ||= CompletedStore.new(File.join(datastore_path, "completed"))
         end
 
-        def completed_queue
-          Web::COMPLETED_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
+        def metadata
+          @metadata ||= Store.new(File.join(datastore_path, "metadata"), thread_safe_reads: false)
         end
 
-        def workers
-          Web::WORKERS[request.get_header("HTTP_X_SPECWRK_RUN")]
+        def run_times
+          @run_times ||= Store.new(File.join(ENV["SPECWRK_OUT"], "run_times"), thread_safe_reads: false)
         end
 
         def worker
-          workers[request.get_header("HTTP_X_SPECWRK_ID")]
+          @worker ||= Store.new(File.join(datastore_path, "workers", request.get_header("HTTP_X_SPECWRK_ID").to_s))
         end
 
         def run_id
@@ -62,7 +99,24 @@ module Specwrk
         end
 
         def run_report_file_path
-          @run_report_file_path ||= File.join(ENV["SPECWRK_OUT"], "#{completed_queue.created_at.strftime("%Y%m%dT%H%M%S")}-report-#{run_id}.json").to_s
+          @run_report_file_path ||= File.join(datastore_path, "#{started_at.strftime("%Y%m%dT%H%M%S")}-report.json").to_s
+        end
+
+        def datastore_path
+          @datastore_path ||= File.join(ENV["SPECWRK_OUT"], run_id).to_s.tap do |path|
+            FileUtils.mkdir_p(path) unless File.directory?(path)
+          end
+        end
+
+        def with_lock
+          Thread.pass until lock_file.flock(File::LOCK_EX)
+          yield
+        ensure
+          lock_file.flock(File::LOCK_UN)
+        end
+
+        def lock_file
+          @lock_file ||= File.open(File.join(datastore_path, "lock"), "a")
         end
       end
 
@@ -70,105 +124,182 @@ module Specwrk
       NotFound = Class.new(Base)
 
       class Health < Base
-        def response
+        def with_response
           [200, {}, []]
         end
       end
 
       class Heartbeat < Base
-        def response
+        def with_response
           ok
         end
       end
 
       class Seed < Base
-        def response
-          pending_queue.synchronize do |pending_queue_hash|
-            unless ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"] && pending_queue_hash.length.positive?
-              examples = payload.map { |hash| [hash[:id], hash] }.to_h
+        def before_lock
+          examples_with_run_times
+        end
 
-              pending_queue.merge_with_previous_run_times!(examples)
+        def with_response
+          pending.clear
+          new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
+          pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
 
-              ok
-            end
-          end
+          pending.merge!(examples_with_run_times)
+          processing.clear
+          completed.clear
 
           ok
         end
-      end
 
-      class Complete < Base
-        def response
-          processing_queue.synchronize do |processing_queue_hash|
+        def examples_with_run_times
+          @examples_with_run_times ||= begin
+            unsorted_examples_with_run_times = []
+            all_ids = payload.map { |example| example[:id] }
+            all_run_times = run_times.multi_read(*all_ids)
+
             payload.each do |example|
-              next unless processing_queue_hash.delete(example[:id])
-              completed_queue[example[:id]] = example
+              run_time = all_run_times[example[:id]]
+
+              unsorted_examples_with_run_times << [example[:id], example.merge(expected_run_time: run_time)]
+            end
+
+            sorted_examples_with_run_times = if sort_by == :timings
+              unsorted_examples_with_run_times.sort_by do |entry|
+                -(entry.last[:expected_run_time] || Float::INFINITY)
+              end
+            else
+              unsorted_examples_with_run_times.sort_by do |entry|
+                entry.last[:file_path]
+              end
             end
+
+            @seeds_run_time_bucket_maximum = run_time_bucket_maximum(all_run_times.values.compact)
+            @examples_with_run_times = sorted_examples_with_run_times.to_h
           end
+        end
+
+        private
 
-          if pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.positive? && ENV["SPECWRK_OUT"]
-            completed_queue.dump_and_write(run_report_file_path)
+        # Average + standard deviation
+        def run_time_bucket_maximum(values)
+          return 0 if values.length.zero?
+
+          mean = values.sum.to_f / values.size
+          variance = values.map { |v| (v - mean)**2 }.sum / values.size
+          (mean + Math.sqrt(variance)).round(2)
+        end
+
+        def sort_by
+          if ENV["SPECWRK_SRV_GROUP_BY"] == "file" || run_times.empty?
+            :file
+          else
+            :timings
           end
+        end
+      end
+
+      class Complete < Base
+        def with_response
+          warn "[DEPRECATED] This endpoint will be retired in favor of CompleteAndPop. Upgrade your clients."
+          completed.merge!(completed_examples)
+          processing.delete(*completed_examples.keys)
 
           ok
         end
-      end
 
-      class Pop < Base
-        def response
-          processing_queue.synchronize do |processing_queue_hash|
-            @examples = pending_queue.shift_bucket
+        private
 
-            @examples.each do |example|
-              processing_queue_hash[example[:id]] = example
-            end
-          end
+        def completed_examples
+          @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
+        end
 
-          if @examples.length.positive?
-            [200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
-          elsif pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.zero?
-            [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
-          elsif completed_queue.length.positive? && processing_queue.length.zero?
-            [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
+        # We don't care about exact values here, just approximate run times are fine
+        # So if we overwrite run times from another process it is nbd
+        def after_lock
+          run_time_data = payload.map { |example| [example[:id], example[:run_time]] }.to_h
+          run_times.merge! run_time_data
+        end
+      end
+
+      class Pop < Base
+        def with_response
+          @examples = pending.shift_bucket
+
+          processing_data = @examples.map { |example| [example[:id], example] }.to_h
+          processing.merge!(processing_data)
+
+          if @examples.any?
+            [200, {"Content-Type" => "application/json"}, [JSON.generate(@examples)]]
+          elsif pending.empty? && processing.empty? && completed.empty?
+            [204, {"Content-Type" => "text/plain"}, ["Waiting for sample to be seeded."]]
+          elsif completed.any? && processing.empty?
+            [410, {"Content-Type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           else
             not_found
           end
         end
       end
 
-      class Report < Base
-        def response
-          if data
-            [200, {"content-type" => "application/json"}, [data]]
+      class CompleteAndPop < Base
+        def with_response
+          completed.merge!(completed_examples)
+          run_times.merge! run_time_data
+          processing.delete(*completed_examples.keys)
+
+          @examples = pending.shift_bucket
+
+          processing_data = @examples.map { |example| [example[:id], example] }.to_h
+          processing.merge!(processing_data)
+
+          if @examples.any?
+            [200, {"Content-Type" => "application/json"}, [JSON.generate(@examples)]]
+          elsif pending.empty? && processing.empty? && completed.empty?
+            [204, {"Content-Type" => "text/plain"}, ["Waiting for sample to be seeded."]]
+          elsif completed.any? && processing.empty?
+            [410, {"Content-Type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           else
-            [404, {"content-type" => "text/plain"}, ["Unable to report on run #{run_id}; no file matching #{"*-report-#{run_id}.json"}"]]
+            not_found
           end
         end
 
         private
 
-        def data
-          return @data if defined? @data
+        def before_lock
+          completed_examples
+          run_time_data
+        end
 
-          return unless most_recent_run_report_file
-          return unless File.exist?(most_recent_run_report_file)
+        def completed_examples
+          @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
+        end
 
-          @data = File.open(most_recent_run_report_file, "r") do |file|
-            file.flock(File::LOCK_SH)
-            file.read
-          end
+        # We don't care about exact values here, just approximate run times are fine
+        # So if we overwrite run times from another process it is nbd
+        def after_lock
+          # run_time_data = payload.map { |example| [example[:id], example[:run_time]] }.to_h
+          # run_times.merge! run_time_data
         end
 
-        def most_recent_run_report_file
-          @most_recent_run_report_file ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], "*-report-#{run_id}.json")).last
+        def run_time_data
+          @run_time_data ||= payload.map { |example| [example[:id], example[:run_time]] }.to_h
+        end
+      end
+
+      class Report < Base
+        def with_response
+          [200, {"Content-Type" => "application/json"}, [JSON.generate(completed.dump)]]
         end
       end
 
       class Shutdown < Base
-        def response
+        def with_response
+          pending.clear
+          processing.clear
+
           interupt! if ENV["SPECWRK_SRV_SINGLE_RUN"]
 
-          [200, {"content-type" => "text/plain"}, ["✌️"]]
+          [200, {"Content-Type" => "text/plain"}, ["✌️"]]
         end
 
         def interupt!

data/lib/specwrk/web.rb

@@ -1,22 +1,6 @@
 # frozen_string_literal: true
 
-require "specwrk/queue"
-
 module Specwrk
   class Web
-    PENDING_QUEUES = Queue.new { |h, key| h[key] = PendingQueue.new }
-    PROCESSING_QUEUES = Queue.new { |h, key| h[key] = Queue.new }
-    COMPLETED_QUEUES = Queue.new { |h, key| h[key] = CompletedQueue.new }
-    WORKERS = Hash.new { |h, key| h[key] = Hash.new { |h, key| h[key] = {} } }
-
-    def self.clear_queues
-      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each(&:clear)
-    end
-
-    def self.clear_run_queues(run)
-      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each do |queue|
-        queue.delete(run)
-      end
-    end
   end
 end

data/lib/specwrk/worker.rb

@@ -69,15 +69,21 @@ module Specwrk
     end
 
     def execute
-      executor.run client.fetch_examples
+      executor.run next_examples
       complete_examples
     rescue UnhandledResponseError => e
-      # If fetching examples fails we can just try again so warn and return
+      # If fetching examples via next_exampels fails we can just try again so warn and return
+      # Expects complete_examples to rescue this error if raised in that method
       warn e.message
     end
 
+    def next_examples
+      return @next_examples if @next_examples&.length&.positive?
+      client.fetch_examples
+    end
+
     def complete_examples
-      client.complete_examples executor.examples
+      @next_examples = client.complete_and_fetch_examples executor.examples
     rescue UnhandledResponseError => e
       # I do not think we should so lightly abandon the completion of executed examples
       # try to complete until successful or terminated

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -9,6 +9,34 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dry-cli
   requirement: !ruby/object:Gem::Requirement
@@ -152,7 +180,6 @@ files:
 - config.ru
 - docker/Dockerfile.server
 - docker/entrypoint.server.sh
-- docker/pitchfork.conf
 - examples/circleci/config.yml
 - examples/github/specwrk-multi-node.yml
 - examples/github/specwrk-single-node.yml
@@ -163,7 +190,8 @@ files:
 - lib/specwrk/client.rb
 - lib/specwrk/hookable.rb
 - lib/specwrk/list_examples.rb
-- lib/specwrk/queue.rb
+- lib/specwrk/store.rb
+- lib/specwrk/store/file_adapter.rb
 - lib/specwrk/version.rb
 - lib/specwrk/web.rb
 - lib/specwrk/web/app.rb
@@ -196,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Parallel rspec test runner from a queue of pending jobs.
 test_files: []

data/docker/pitchfork.conf

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-worker_processes 1
-listen "localhost:3000", backlog: 2048
-timeout 301

data/lib/specwrk/queue.rb

@@ -1,224 +0,0 @@
-# frozen_string_literal: true
-
-require "time"
-require "json"
-
-module Specwrk
-  # Thread-safe Hash access
-  class Queue
-    attr_reader :created_at
-
-    def initialize(hash = {})
-      @created_at = Time.now
-
-      if block_given?
-        @mutex = Monitor.new # Reentrant locking is required here
-        # It's possible to enter the proc from two threads, so we need to ||= in case
-        # one thread has set a value prior to the yield.
-        hash.default_proc = proc { |h, key| @mutex.synchronize { yield(h, key) } }
-      end
-
-      @mutex ||= Mutex.new # Monitor is up-to 20% slower than Mutex, so if no block is given, use a mutex
-      @hash = hash
-    end
-
-    def synchronize(&blk)
-      if @mutex.owned?
-        yield(@hash)
-      else
-        @mutex.synchronize { yield(@hash) }
-      end
-    end
-
-    def method_missing(name, *args, &block)
-      if @hash.respond_to?(name)
-        @mutex.synchronize { @hash.public_send(name, *args, &block) }
-      else
-        super
-      end
-    end
-
-    def respond_to_missing?(name, include_private = false)
-      @hash.respond_to?(name, include_private) || super
-    end
-  end
-
-  class PendingQueue < Queue
-    def shift_bucket
-      return bucket_by_file unless previous_run_times
-
-      case ENV["SPECWRK_SRV_GROUP_BY"]
-      when "file"
-        bucket_by_file
-      else
-        bucket_by_timings
-      end
-    end
-
-    def run_time_bucket_threshold
-      return 1 unless previous_run_times
-
-      previous_run_times.dig(:meta, :average_run_time)
-    end
-
-    def previous_run_times
-      return unless ENV["SPECWRK_OUT"]
-
-      @previous_run_times ||= begin
-        return unless previous_run_times_file_path
-        return unless File.exist? previous_run_times_file_path
-
-        raw_data = File.open(previous_run_times_file_path, "r") do |file|
-          file.flock(File::LOCK_SH)
-          file.read
-        end
-
-        @previous_run_times = JSON.parse(raw_data, symbolize_names: true)
-      rescue JSON::ParserError => e
-        warn "#{e.inspect} in file #{previous_run_times_file_path}"
-        nil
-      end
-    end
-
-    def merge_with_previous_run_times!(h2)
-      synchronize do
-        h2.each { |_id, example| merge_example(example) }
-
-        # Sort by exepcted run time, slowest to fastest
-        @hash = @hash.sort_by { |_, example| example[:expected_run_time] }.reverse.to_h
-      end
-    end
-
-    private
-
-    # We want the most recently modified run time file
-    # report files are prefixed with a timestamp, and Dir.glob should order
-    # alphanumericly
-    def previous_run_times_file_path
-      return unless ENV["SPECWRK_OUT"]
-
-      @previous_run_times_file_path ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], "*-report-*.json")).last
-    end
-
-    # Take elements from the hash where the file_path is the same
-    def bucket_by_file
-      bucket = []
-
-      @mutex.synchronize do
-        key = @hash.keys.first
-        break if key.nil?
-
-        file_path = @hash[key][:file_path]
-        @hash.each do |id, example|
-          next unless example[:file_path] == file_path
-
-          bucket << example
-          @hash.delete id
-        end
-      end
-
-      bucket
-    end
-
-    # Take elements from the hash until the average runtime bucket has filled
-    def bucket_by_timings
-      bucket = []
-
-      @mutex.synchronize do
-        estimated_run_time_total = 0
-
-        while estimated_run_time_total < run_time_bucket_threshold
-          key = @hash.keys.first
-          break if key.nil?
-
-          estimated_run_time_total += @hash.dig(key, :expected_run_time)
-          break if estimated_run_time_total > run_time_bucket_threshold && bucket.length.positive?
-
-          bucket << @hash[key]
-          @hash.delete key
-        end
-      end
-
-      bucket
-    end
-
-    # Ensure @mutex is held when calling this method
-    def merge_example(example)
-      return if @hash.key? example[:id]
-      return if @hash.key? example[:file_path]
-
-      @hash[example[:id]] = if previous_run_times
-        example.merge!(
-          expected_run_time: previous_run_times.dig(:examples, example[:id].to_sym, :run_time) || 99999.9 # run "unknown" files first
-        )
-      else
-        example.merge!(
-          expected_run_time: 99999.9 # run "unknown" files first
-        )
-      end
-    end
-  end
-
-  class CompletedQueue < Queue
-    def dump_and_write(path)
-      write_output_to(path, dump)
-    end
-
-    def dump
-      @mutex.synchronize do
-        @run_times = []
-        @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
-        @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
-
-        @output = {
-          file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
-          meta: {failures: 0, passes: 0, pending: 0},
-          examples: {}
-        }
-
-        @hash.values.each { |example| calculate(example) }
-
-        @output[:meta][:total_run_time] = @run_times.sum
-        @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
-        @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
-        @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
-
-        @output
-      end
-    end
-
-    private
-
-    def calculate(example)
-      @run_times << example[:run_time]
-      @output[:file_totals][example[:file_path]] += example[:run_time]
-
-      started_at = Time.parse(example[:started_at])
-      finished_at = Time.parse(example[:finished_at])
-
-      @first_started_at = started_at if started_at < @first_started_at
-      @last_finished_at = finished_at if finished_at > @last_finished_at
-
-      case example[:status]
-      when "passed"
-        @output[:meta][:passes] += 1
-      when "failed"
-        @output[:meta][:failures] += 1
-      when "pending"
-        @output[:meta][:pending] += 1
-      end
-
-      @output[:examples][example[:id]] = example
-    end
-
-    def write_output_to(path, output)
-      File.open(path, "w") do |file|
-        file.flock(File::LOCK_EX)
-
-        file.write JSON.pretty_generate(output)
-
-        file.flock(File::LOCK_UN)
-      end
-    end
-  end
-end