specwrk 0.7.1 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 690e5d97390d650309a664688c09c777308ad8a0e8c52ef853fd7893a23d73f3
-  data.tar.gz: b9dfe358291cad4ea4e1a40bebc82305c9d98dc05f2620ee7268c2a60990dbc0
+  metadata.gz: 657cd8e611ea7ea6344dc5bd540892e69d801ea5afd410b1be2e7a6d3f73658e
+  data.tar.gz: c066775a6f57452cd55c49eec7604b8e264881ad085d7847ff176240a9ecbe05
 SHA512:
-  metadata.gz: d5fe7fa3b2cdcf27b542a7a75a49168e7dba16f6c92982d77fd66a3d60776a12f4d5e500a3e34945d18913e7b3f4493906116bed47266f115ddffed29373995e
-  data.tar.gz: b64f866c748808b8e7e5ba788c14a33fbd57f9bf5f8589761ec36fc3745c25cea95bdef291802137530a387577a816128a09dac34db6107133204efe51d34fee
+  metadata.gz: 076a41a98119ccfe0108c76c32341d8cbf6642da3f37f80276a61ed885fbca9ddaf31c597b8462ddeb608c67d7e8787f5cefd606cb4b977aeef715e9a5cdb0bb
+  data.tar.gz: 39891db9a0400196ddaba587e7014aa00e485c65ce37efc997998f854dd98d8cabfef39806657bec15e71c43a29743dda4081b35c2022a917a5503e4e161cea4

data/docker/entrypoint.server.sh

@@ -4,4 +4,4 @@ export THRUSTER_HTTP_PORT=${PORT:-5138}
 export THRUSTER_TARGET_PORT=3000
 export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-305}
 
-exec thrust puma --workers 0 --bind tcp://127.0.0.1:3000 --threads ${PUMA_THREADS:-1}
+exec thrust puma --workers 0 --bind tcp://127.0.0.1:3000 --threads ${PUMA_THREADS:-1} --workers ${PUMA_WORKERS:-0}

data/lib/specwrk/store/file_adapter.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require "json"
+require "base64"
+
+require "specwrk/store"
+
+module Specwrk
+  class Store
+    class FileAdapter
+      EXT = ".wrk.json"
+
+      THREAD_POOL = Class.new do
+        @work_queue = Queue.new
+
+        @threads = Array.new(ENV.fetch("SPECWRK_SRV_FILE_ADAPTER_THREAD_COUNT", "4").to_i) do
+          Thread.new do
+            loop do
+              @work_queue.pop.call
+            end
+          end
+        end
+
+        class << self
+          def schedule(&blk)
+            @work_queue.push blk
+          end
+        end
+      end
+
+      def initialize(path)
+        @path = path
+        FileUtils.mkdir_p(@path)
+      end
+
+      def [](key)
+        content = read(key.to_s)
+        return unless content
+
+        JSON.parse(content, symbolize_names: true)
+      end
+
+      def []=(key, value)
+        key_string = key.to_s
+        if value.nil?
+          delete(key_string)
+        else
+          filename = filename_for_key(key_string)
+          write(filename, JSON.generate(value))
+          known_key_pairs[key_string] = filename
+        end
+      end
+
+      def keys
+        known_key_pairs.keys
+      end
+
+      def clear
+        FileUtils.rm_rf(@path)
+        FileUtils.mkdir_p(@path)
+
+        @known_key_pairs = nil
+      end
+
+      def delete(*keys)
+        filenames = keys.map { |key| known_key_pairs[key] }.compact
+
+        FileUtils.rm_f(filenames)
+
+        keys.each { |key| known_key_pairs.delete(key) }
+      end
+
+      def merge!(h2)
+        multi_write(h2)
+      end
+
+      def multi_read(*read_keys)
+        known_key_pairs # precache before each thread tries to look them up
+
+        result_queue = Queue.new
+
+        read_keys.each do |key|
+          THREAD_POOL.schedule do
+            result_queue.push([key.to_s, read(key)])
+          end
+        end
+
+        Thread.pass until result_queue.length == read_keys.length
+
+        results = {}
+        until result_queue.empty?
+          result = result_queue.pop
+          next if result.last.nil?
+
+          results[result.first] = JSON.parse(result.last, symbolize_names: true)
+        end
+
+        read_keys.map { |key| [key.to_s, results[key.to_s]] if results.key?(key.to_s) }.compact.to_h # respect order requested in the returned hash
+      end
+
+      def multi_write(hash)
+        known_key_pairs # precache before each thread tries to look them up
+
+        result_queue = Queue.new
+
+        hash_with_filenames = hash.map { |key, value| [key.to_s, [filename_for_key(key.to_s), value]] }.to_h
+        hash_with_filenames.each do |key, (filename, value)|
+          content = JSON.generate(value)
+
+          THREAD_POOL.schedule do
+            result_queue << write(filename, content)
+          end
+        end
+
+        Thread.pass until result_queue.length == hash.length
+        hash_with_filenames.each { |key, (filename, _value)| known_key_pairs[key] = filename }
+      end
+
+      def empty?
+        Dir.empty? @path
+      end
+
+      private
+
+      def write(filename, content)
+        tmp_filename = [filename, "tmp"].join(".")
+
+        File.binwrite(tmp_filename, content)
+
+        FileUtils.mv tmp_filename, filename
+        true
+      end
+
+      def read(key)
+        File.read(known_key_pairs[key]) if known_key_pairs.key? key
+      end
+
+      def filename_for_key(key)
+        File.join(
+          @path,
+          [
+            counter_prefix(key),
+            encode_key(key)
+          ].join("_")
+        ) + EXT
+      end
+
+      def counter_prefix(key)
+        count = keys.index(key) || counter.tap { @counter += 1 }
+
+        "%012d" % count
+      end
+
+      def counter
+        @counter ||= keys.length
+      end
+
+      def encode_key(key)
+        Base64.urlsafe_encode64(key).delete("=")
+      end
+
+      def decode_key(key)
+        encoded_key_part = File.basename(key).delete_suffix(EXT).split(/\A\d+_/).last
+        padding_count = (4 - encoded_key_part.length % 4) % 4
+
+        Base64.urlsafe_decode64(encoded_key_part + ("=" * padding_count))
+      end
+
+      def known_key_pairs
+        @known_key_pairs ||= Dir.entries(@path).sort.map do |filename|
+          next if filename.start_with? "."
+          next unless filename.end_with? EXT
+
+          file_path = File.join(@path, filename)
+          [decode_key(filename), file_path]
+        end.compact.to_h
+      end
+    end
+  end
+end

data/lib/specwrk/store.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require "time"
+require "json"
+
+require "specwrk/store/file_adapter"
+
+module Specwrk
+  class Store
+    MUTEXES = {}
+    MUTEXES_MUTEX = Mutex.new # 🐢🐢🐢🐢
+
+    class << self
+      def mutex_for(path)
+        MUTEXES_MUTEX.synchronize do
+          MUTEXES[path] ||= Mutex.new
+        end
+      end
+    end
+
+    def initialize(path, thread_safe_reads: true)
+      @path = path
+      @thread_safe_reads = thread_safe_reads
+    end
+
+    def [](key)
+      sync(thread_safe: thread_safe_reads) { adapter[key.to_s] }
+    end
+
+    def multi_read(*keys)
+      sync(thread_safe: thread_safe_reads) { adapter.multi_read(*keys) }
+    end
+
+    def []=(key, value)
+      sync do
+        adapter[key.to_s] = value
+      end
+    end
+
+    def keys
+      all_keys = sync(thread_safe: thread_safe_reads) do
+        adapter.keys
+      end
+
+      all_keys.reject { |k| k.start_with? "____" }
+    end
+
+    def length
+      keys.length
+    end
+
+    def any?
+      !empty?
+    end
+
+    def empty?
+      sync(thread_safe: thread_safe_reads) do
+        adapter.empty?
+      end
+    end
+
+    def delete(*keys)
+      sync { adapter.delete(*keys) }
+    end
+
+    def merge!(h2)
+      h2.transform_keys!(&:to_s)
+      sync { adapter.merge!(h2) }
+    end
+
+    def clear
+      sync { adapter.clear }
+    end
+
+    def to_h
+      sync(thread_safe: thread_safe_reads) do
+        adapter.multi_read(*keys).transform_keys!(&:to_sym)
+      end
+    end
+
+    def inspect
+      reload.to_h.dup
+    end
+
+    # Bypass any cached values. Helpful when you have two instances
+    # of the same store where one mutates data and the other needs to check
+    # on the status of that data (i.e. endpoint tests)
+    def reload
+      @adapter = nil
+      self
+    end
+
+    private
+
+    attr_reader :thread_safe_reads
+
+    def sync(thread_safe: true)
+      if !thread_safe || mutex.owned?
+        yield
+      else
+        mutex.synchronize { yield }
+      end
+    end
+
+    def adapter
+      @adapter ||= FileAdapter.new(@path)
+    end
+
+    def mutex
+      @mutex ||= self.class.mutex_for(@path)
+    end
+  end
+
+  class PendingStore < Store
+    RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
+
+    def run_time_bucket_maximum=(val)
+      @run_time_bucket_maximum = self[RUN_TIME_BUCKET_MAXIMUM_KEY] = val
+    end
+
+    def run_time_bucket_maximum
+      @run_time_bucket_maximum ||= self[RUN_TIME_BUCKET_MAXIMUM_KEY]
+    end
+
+    def shift_bucket
+      sync do
+        return bucket_by_file unless run_time_bucket_maximum&.positive?
+
+        case ENV["SPECWRK_SRV_GROUP_BY"]
+        when "file"
+          bucket_by_file
+        else
+          bucket_by_timings
+        end
+      end
+    end
+
+    private
+
+    # Take elements from the hash where the file_path is the same
+    # Expects that the examples were merged in order of filename
+    def bucket_by_file
+      bucket = []
+      consumed_keys = []
+
+      all_keys = keys
+      key = all_keys.first
+      return [] if key.nil?
+
+      file_path = self[key][:file_path]
+
+      catch(:full) do
+        all_keys.each_slice(24).each do |key_group|
+          examples = multi_read(*key_group)
+
+          examples.each do |key, example|
+            throw :full unless example[:file_path] == file_path
+
+            bucket << example
+            consumed_keys << key
+          end
+        end
+      end
+
+      delete(*consumed_keys)
+      bucket
+    end
+
+    # Take elements from the hash until the average runtime bucket has filled
+    def bucket_by_timings
+      bucket = []
+      consumed_keys = []
+
+      estimated_run_time_total = 0
+
+      catch(:full) do
+        keys.each_slice(25).each do |key_group|
+          examples = multi_read(*key_group)
+
+          examples.each do |key, example|
+            estimated_run_time_total += example[:expected_run_time] || run_time_bucket_maximum
+            throw :full if estimated_run_time_total > run_time_bucket_maximum && bucket.length.positive?
+
+            bucket << example
+            consumed_keys << key
+          end
+        end
+      end
+
+      delete(*consumed_keys)
+      bucket
+    end
+  end
+
+  class CompletedStore < Store
+    def dump
+      @run_times = []
+      @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
+      @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
+
+      @output = {
+        file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
+        meta: {failures: 0, passes: 0, pending: 0},
+        examples: {}
+      }
+
+      to_h.values.each { |example| calculate(example) }
+
+      @output[:meta][:total_run_time] = @run_times.sum
+      @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
+      @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
+      @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
+
+      @output
+    end
+
+    private
+
+    def calculate(example)
+      @run_times << example[:run_time]
+      @output[:file_totals][example[:file_path]] += example[:run_time]
+
+      started_at = Time.parse(example[:started_at])
+      finished_at = Time.parse(example[:finished_at])
+
+      @first_started_at = started_at if started_at < @first_started_at
+      @last_finished_at = finished_at if finished_at > @last_finished_at
+
+      case example[:status]
+      when "passed"
+        @output[:meta][:passes] += 1
+      when "failed"
+        @output[:meta][:failures] += 1
+      when "pending"
+        @output[:meta][:pending] += 1
+      end
+
+      @output[:examples][example[:id]] = example
+    end
+  end
+end

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.7.1"
+  VERSION = "0.9.1"
 end

data/lib/specwrk/web/endpoints.rb

@@ -2,41 +2,38 @@
 
 require "json"
 
+require "specwrk/store"
+
 module Specwrk
   class Web
     module Endpoints
       class Base
-        attr_reader :pending_queue, :processing_queue, :completed_queue, :workers, :started_at
+        attr_reader :started_at
 
         def initialize(request)
           @request = request
         end
 
         def response
+          before_lock
+
           return with_response unless run_id # No run_id, no datastore usage in the endpoint
 
-          datastore.with_lock do |db|
-            @started_at = if db[:started_at]
-              Time.parse(db[:started_at])
-            else
-              db[:started_at] = Time.now
-            end
+          payload # parse the payload before any locking
 
-            @pending_queue = PendingQueue.new.merge!(db[:pending] || {})
-            @processing_queue = Queue.new.merge!(db[:processing] || {})
-            @completed_queue = CompletedQueue.new.merge!(db[:completed] || {})
-            @workers = db[:workers] ||= {}
+          worker[:first_seen_at] ||= Time.now.iso8601
+          worker[:last_seen_at] = Time.now.iso8601
 
-            worker[:first_seen_at] ||= Time.now
-            worker[:last_seen_at] = Time.now
+          final_response = with_lock do
+            started_at = metadata[:started_at] ||= Time.now.iso8601
+            @started_at = Time.parse(started_at)
 
-            with_response.tap do
-              db[:pending] = pending_queue.to_h
-              db[:processing] = processing_queue.to_h
-              db[:completed] = completed_queue.to_h
-              db[:workers] = workers.to_h
-            end
+            with_response
           end
+
+          after_lock
+
+          final_response
         end
 
         def with_response
@@ -47,6 +44,12 @@ module Specwrk
 
         attr_reader :request
 
+        def before_lock
+        end
+
+        def after_lock
+        end
+
         def not_found
           [404, {"Content-Type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
         end
@@ -56,6 +59,10 @@ module Specwrk
         end
 
         def payload
+          return unless request.content_type&.start_with?("application/json")
+          return unless request.post? || request.put? || request.delete?
+          return if body.empty?
+
           @payload ||= JSON.parse(body, symbolize_names: true)
         end
 
@@ -63,8 +70,28 @@ module Specwrk
           @body ||= request.body.read
         end
 
+        def pending
+          @pending ||= PendingStore.new(File.join(datastore_path, "pending"))
+        end
+
+        def processing
+          @processing ||= Store.new(File.join(datastore_path, "processing"))
+        end
+
+        def completed
+          @completed ||= CompletedStore.new(File.join(datastore_path, "completed"))
+        end
+
+        def metadata
+          @metadata ||= Store.new(File.join(datastore_path, "metadata"), thread_safe_reads: false)
+        end
+
+        def run_times
+          @run_times ||= Store.new(File.join(ENV["SPECWRK_OUT"], "run_times"), thread_safe_reads: false)
+        end
+
         def worker
-          workers[request.get_header("HTTP_X_SPECWRK_ID")] ||= {}
+          @worker ||= Store.new(File.join(datastore_path, "workers", request.get_header("HTTP_X_SPECWRK_ID").to_s))
         end
 
         def run_id
@@ -72,11 +99,24 @@ module Specwrk
         end
 
         def run_report_file_path
-          @run_report_file_path ||= File.join(ENV["SPECWRK_OUT"], run_id, "#{started_at.strftime("%Y%m%dT%H%M%S")}-report.json").to_s
+          @run_report_file_path ||= File.join(datastore_path, "#{started_at.strftime("%Y%m%dT%H%M%S")}-report.json").to_s
         end
 
-        def datastore
-          Web.datastore[File.join(ENV["SPECWRK_OUT"], run_id, "queues.json").to_s]
+        def datastore_path
+          @datastore_path ||= File.join(ENV["SPECWRK_OUT"], run_id).to_s.tap do |path|
+            FileUtils.mkdir_p(path) unless File.directory?(path)
+          end
+        end
+
+        def with_lock
+          Thread.pass until lock_file.flock(File::LOCK_EX)
+          yield
+        ensure
+          lock_file.flock(File::LOCK_UN)
+        end
+
+        def lock_file
+          @lock_file ||= File.open(File.join(datastore_path, "lock"), "a")
         end
       end
 
@@ -96,45 +136,109 @@ module Specwrk
       end
 
       class Seed < Base
+        def before_lock
+          examples_with_run_times if persist_seeds?
+        end
+
         def with_response
-          if ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"].nil? || pending_queue.length.zero?
-            examples = payload.map { |hash| [hash[:id], hash] }.to_h
-            pending_queue.merge_with_previous_run_times!(examples)
+          if persist_seeds?
+            new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
+            pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
+
+            pending.merge!(examples_with_run_times)
           end
 
+          processing.clear
+          completed.clear
+
           ok
         end
-      end
 
-      class Complete < Base
-        def with_response
-          payload.each do |example|
-            next unless processing_queue.delete(example[:id].to_sym)
-            completed_queue[example[:id].to_sym] = example
+        def examples_with_run_times
+          @examples_with_run_times ||= begin
+            unsorted_examples_with_run_times = []
+            all_ids = payload.map { |example| example[:id] }
+            all_run_times = run_times.multi_read(*all_ids)
+
+            payload.each do |example|
+              run_time = all_run_times[example[:id]]
+
+              unsorted_examples_with_run_times << [example[:id], example.merge(expected_run_time: run_time)]
+            end
+
+            sorted_examples_with_run_times = if sort_by == :timings
+              unsorted_examples_with_run_times.sort_by do |entry|
+                -(entry.last[:expected_run_time] || Float::INFINITY)
+              end
+            else
+              unsorted_examples_with_run_times.sort_by do |entry|
+                entry.last[:file_path]
+              end
+            end
+
+            @seeds_run_time_bucket_maximum = run_time_bucket_maximum(all_run_times.values.compact)
+            @examples_with_run_times = sorted_examples_with_run_times.to_h
           end
+        end
+
+        private
+
+        # Average + standard deviation
+        def run_time_bucket_maximum(values)
+          return 0 if values.length.zero?
+
+          mean = values.sum.to_f / values.size
+          variance = values.map { |v| (v - mean)**2 }.sum / values.size
+          (mean + Math.sqrt(variance)).round(2)
+        end
 
-          if pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.positive? && ENV["SPECWRK_OUT"]
-            completed_queue.dump_and_write(run_report_file_path)
-            FileUtils.ln_sf(run_report_file_path, File.join(ENV["SPECWRK_OUT"], "report.json"))
+        def persist_seeds?
+          ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"].nil? || pending.empty?
+        end
+
+        def sort_by
+          if ENV["SPECWRK_SRV_GROUP_BY"] == "file" || run_times.empty?
+            :file
+          else
+            :timings
           end
+        end
+      end
+
+      class Complete < Base
+        def with_response
+          completed.merge!(completed_examples)
+          processing.delete(*completed_examples.keys)
 
           ok
         end
+
+        private
+
+        def completed_examples
+          @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
+        end
+
+        # We don't care about exact values here, just approximate run times are fine
+        # So if we overwrite run times from another process it is nbd
+        def after_lock
+          run_time_data = payload.map { |example| [example[:id], example[:run_time]] }.to_h
+          run_times.merge! run_time_data
+        end
       end
 
       class Pop < Base
         def with_response
-          @examples = pending_queue.shift_bucket
+          @examples = pending.shift_bucket
 
-          @examples.each do |example|
-            processing_queue[example[:id]] = example
-          end
+          processing_data = @examples.map { |example| [example[:id], example] }.to_h
+          processing.merge!(processing_data)
 
-          if @examples.length.positive?
+          if @examples.any?
             [200, {"Content-Type" => "application/json"}, [JSON.generate(@examples)]]
-          elsif pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.zero?
+          elsif pending.empty? && processing.empty? && completed.empty?
             [204, {"Content-Type" => "text/plain"}, ["Waiting for sample to be seeded."]]
-          elsif completed_queue.length.positive? && processing_queue.length.zero?
+          elsif completed.any? && processing.empty?
             [410, {"Content-Type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           else
             not_found
@@ -144,34 +248,15 @@ module Specwrk
 
       class Report < Base
         def with_response
-          if data
-            [200, {"Content-Type" => "application/json"}, [data]]
-          else
-            [404, {"Content-Type" => "text/plain"}, ["Unable to report on run #{run_id}; no file matching #{"*-report-#{run_id}.json"}"]]
-          end
-        end
-
-        private
-
-        def data
-          return @data if defined? @data
-
-          return unless most_recent_run_report_file
-          return unless File.exist?(most_recent_run_report_file)
-
-          @data = File.open(most_recent_run_report_file, "r") do |file|
-            file.flock(File::LOCK_SH)
-            file.read
-          end
-        end
-
-        def most_recent_run_report_file
-          @most_recent_run_report_file ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], run_id, "*-report.json")).last
+          [200, {"Content-Type" => "application/json"}, [JSON.generate(completed.dump)]]
         end
       end
 
       class Shutdown < Base
         def with_response
+          pending.clear
+          processing.clear
+
           interupt! if ENV["SPECWRK_SRV_SINGLE_RUN"]
 
           [200, {"Content-Type" => "text/plain"}, ["✌️"]]

data/lib/specwrk/web.rb

@@ -1,14 +1,6 @@
 # frozen_string_literal: true
 
-require "specwrk/queue"
-require "specwrk/filestore"
-
 module Specwrk
   class Web
-    class << self
-      def datastore
-        Filestore
-      end
-    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.9.1
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -9,6 +9,34 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dry-cli
   requirement: !ruby/object:Gem::Requirement
@@ -160,10 +188,10 @@ files:
 - lib/specwrk/cli.rb
 - lib/specwrk/cli_reporter.rb
 - lib/specwrk/client.rb
-- lib/specwrk/filestore.rb
 - lib/specwrk/hookable.rb
 - lib/specwrk/list_examples.rb
-- lib/specwrk/queue.rb
+- lib/specwrk/store.rb
+- lib/specwrk/store/file_adapter.rb
 - lib/specwrk/version.rb
 - lib/specwrk/web.rb
 - lib/specwrk/web/app.rb
@@ -196,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Parallel rspec test runner from a queue of pending jobs.
 test_files: []

data/lib/specwrk/filestore.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-require "fileutils"
-
-module Specwrk
-  class Filestore
-    @mutexes = {}
-    @mutexes_mutex = Mutex.new # 🐢🐢🐢🐢
-
-    class << self
-      def [](path)
-        new(path)
-      end
-
-      def mutex_for(path)
-        @mutexes_mutex.synchronize do
-          @mutexes[path] ||= Mutex.new
-        end
-      end
-    end
-
-    def initialize(path)
-      @path = path
-      @tmpfile_path = @path + ".tmp"
-      @lock_path = @path + ".lock"
-
-      FileUtils.mkdir_p File.dirname(@path)
-      File.open(@path, "a") {} # multi-process and multi-thread safe touch
-    end
-
-    def with_lock
-      self.class.mutex_for(@path).synchronize do
-        lock_file.flock(File::LOCK_EX)
-
-        hash = read
-        result = yield(hash)
-
-        # Will truncate if already exists
-        File.open(@tmpfile_path, "w") do |tmpfile|
-          tmpfile.write(hash.to_json)
-          tmpfile.fsync
-          tmpfile.close
-        end
-
-        File.rename(@tmpfile_path, @path)
-        result
-      ensure
-        lock_file.flock(File::LOCK_UN)
-      end
-    end
-
-    private
-
-    def lock_file
-      @lock_file ||= File.open(@lock_path, "w")
-    end
-
-    def read
-      JSON.parse(File.read(@path), symbolize_names: true)
-    rescue JSON::ParserError
-      {}
-    end
-  end
-end

data/lib/specwrk/queue.rb

@@ -1,180 +0,0 @@
-# frozen_string_literal: true
-
-require "time"
-require "json"
-
-module Specwrk
-  Queue = Class.new(Hash)
-
-  class PendingQueue < Queue
-    def shift_bucket
-      return bucket_by_file unless previous_run_times
-
-      case ENV["SPECWRK_SRV_GROUP_BY"]
-      when "file"
-        bucket_by_file
-      else
-        bucket_by_timings
-      end
-    end
-
-    def run_time_bucket_threshold
-      return 1 unless previous_run_times
-
-      previous_run_times.dig(:meta, :average_run_time)
-    end
-
-    def previous_run_times
-      return unless ENV["SPECWRK_OUT"]
-
-      @previous_run_times ||= begin
-        return unless previous_run_times_file_path
-        return unless File.exist? previous_run_times_file_path
-
-        raw_data = File.open(previous_run_times_file_path, "r") do |file|
-          file.flock(File::LOCK_SH)
-          file.read
-        end
-
-        @previous_run_times = JSON.parse(raw_data, symbolize_names: true)
-      rescue JSON::ParserError => e
-        warn "#{e.inspect} in file #{previous_run_times_file_path}"
-        nil
-      end
-    end
-
-    def merge_with_previous_run_times!(h2)
-      h2.each { |_id, example| merge_example(example) }
-
-      # Sort by exepcted run time, slowest to fastest
-      new_h = sort_by { |_, example| example[:expected_run_time] }.reverse.to_h
-      clear
-      merge!(new_h)
-    end
-
-    private
-
-    # We want the most recently modified run time file
-    # report files are prefixed with a timestamp, and Dir.glob should order
-    # alphanumericly
-    def previous_run_times_file_path
-      return unless ENV["SPECWRK_OUT"]
-
-      @previous_run_times_file_path ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], "*-report-*.json")).last
-    end
-
-    # Take elements from the hash where the file_path is the same
-    def bucket_by_file
-      bucket = []
-
-      key = keys.first
-      return [] if key.nil?
-
-      file_path = self[key][:file_path]
-      each do |id, example|
-        next unless example[:file_path] == file_path
-
-        bucket << example
-        delete id
-      end
-
-      bucket
-    end
-
-    # Take elements from the hash until the average runtime bucket has filled
-    def bucket_by_timings
-      bucket = []
-
-      estimated_run_time_total = 0
-
-      while estimated_run_time_total < run_time_bucket_threshold
-        key = keys.first
-        break if key.nil?
-
-        estimated_run_time_total += dig(key, :expected_run_time)
-        break if estimated_run_time_total > run_time_bucket_threshold && bucket.length.positive?
-
-        bucket << self[key]
-        delete key
-      end
-
-      bucket
-    end
-
-    def merge_example(example)
-      return if key? example[:id]
-      return if key? example[:file_path]
-
-      self[example[:id]] = if previous_run_times
-        example.merge!(
-          expected_run_time: previous_run_times.dig(:examples, example[:id].to_sym, :run_time) || 99999.9 # run "unknown" files first
-        )
-      else
-        example.merge!(
-          expected_run_time: 99999.9 # run "unknown" files first
-        )
-      end
-    end
-  end
-
-  class CompletedQueue < Queue
-    def dump_and_write(path)
-      write_output_to(path, dump)
-    end
-
-    def dump
-      @run_times = []
-      @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
-      @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
-
-      @output = {
-        file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
-        meta: {failures: 0, passes: 0, pending: 0},
-        examples: {}
-      }
-
-      values.each { |example| calculate(example) }
-
-      @output[:meta][:total_run_time] = @run_times.sum
-      @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
-      @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
-      @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
-
-      @output
-    end
-
-    private
-
-    def calculate(example)
-      @run_times << example[:run_time]
-      @output[:file_totals][example[:file_path]] += example[:run_time]
-
-      started_at = Time.parse(example[:started_at])
-      finished_at = Time.parse(example[:finished_at])
-
-      @first_started_at = started_at if started_at < @first_started_at
-      @last_finished_at = finished_at if finished_at > @last_finished_at
-
-      case example[:status]
-      when "passed"
-        @output[:meta][:passes] += 1
-      when "failed"
-        @output[:meta][:failures] += 1
-      when "pending"
-        @output[:meta][:pending] += 1
-      end
-
-      @output[:examples][example[:id]] = example
-    end
-
-    def write_output_to(path, output)
-      File.open(path, "w") do |file|
-        file.flock(File::LOCK_EX)
-
-        file.write JSON.pretty_generate(output)
-
-        file.flock(File::LOCK_UN)
-      end
-    end
-  end
-end