specwrk 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 690e5d97390d650309a664688c09c777308ad8a0e8c52ef853fd7893a23d73f3
-  data.tar.gz: b9dfe358291cad4ea4e1a40bebc82305c9d98dc05f2620ee7268c2a60990dbc0
+  metadata.gz: 2122e3e3925fdf1c12b6d33899c52ba1b8287e46e7518754ae0ca8188ac149ce
+  data.tar.gz: 89579242f18a5f17aac60ac52664e1197cf61ef35228d142298b4ad250057a25
 SHA512:
-  metadata.gz: d5fe7fa3b2cdcf27b542a7a75a49168e7dba16f6c92982d77fd66a3d60776a12f4d5e500a3e34945d18913e7b3f4493906116bed47266f115ddffed29373995e
-  data.tar.gz: b64f866c748808b8e7e5ba788c14a33fbd57f9bf5f8589761ec36fc3745c25cea95bdef291802137530a387577a816128a09dac34db6107133204efe51d34fee
+  metadata.gz: c357e649df3ad4533290c7e53be49d1f4a3bf86870b5cc01ed4bc2219393bc026d4056e65c82b21c9f251724fe3adcebf6efda7a2643b85d873e4ede7efe17d3
+  data.tar.gz: 1aea05a1f33cd4bfeddbbe76d7d97c7dd3535ef4d1c24b5116042bcfe37201acc7a70c0839a6ef28ba9a5ecd9380535a7007c8a118e5277fc8d798899d7f28e1

data/docker/Dockerfile.server

@@ -1,11 +1,6 @@
 FROM ruby:3.4-alpine
 
-RUN apk add --no-cache \
-      build-base \
-      ruby-dev \
-      linux-headers \
-      zlib-dev \
-      libffi-dev
+RUN apk add --no-cache build-base 
 
 WORKDIR /app
 
@@ -13,14 +8,15 @@ RUN mkdir .specwrk/
 
 ARG SPECWRK_SRV_PORT=5138
 ARG SPECWRK_VERSION=latest
-ARG GEMFILE=specwrk-$SPECWRK_VERSION.gem
+ARG GEM_FILE=specwrk-$SPECWRK_VERSION.gem
 
-COPY $GEMFILE ./
-RUN gem install ./$GEMFILE --no-document
-RUN rm ./$GEMFILE
+COPY $GEM_FILE ./
+RUN gem install ./$GEM_FILE --no-document
+RUN rm ./$GEM_FILE
 
-RUN gem install puma thruster
+RUN gem install pitchfork thruster
 COPY config.ru ./
+COPY docker/pitchfork.conf ./
 
 COPY docker/entrypoint.server.sh /usr/local/bin/entrypoint
 RUN chmod +x /usr/local/bin/entrypoint

data/docker/entrypoint.server.sh

@@ -2,6 +2,6 @@
 
 export THRUSTER_HTTP_PORT=${PORT:-5138}
 export THRUSTER_TARGET_PORT=3000
-export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-305}
+export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-300}
 
-exec thrust puma --workers 0 --bind tcp://127.0.0.1:3000 --threads ${PUMA_THREADS:-1}
+exec thrust pitchfork -c pitchfork.conf

data/docker/pitchfork.conf

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+worker_processes 1
+listen "localhost:3000", backlog: 2048
+timeout 301

data/lib/specwrk/queue.rb

@@ -4,7 +4,44 @@ require "time"
 require "json"
 
 module Specwrk
-  Queue = Class.new(Hash)
+  # Thread-safe Hash access
+  class Queue
+    attr_reader :created_at
+
+    def initialize(hash = {})
+      @created_at = Time.now
+
+      if block_given?
+        @mutex = Monitor.new # Reentrant locking is required here
+        # It's possible to enter the proc from two threads, so we need to ||= in case
+        # one thread has set a value prior to the yield.
+        hash.default_proc = proc { |h, key| @mutex.synchronize { yield(h, key) } }
+      end
+
+      @mutex ||= Mutex.new # Monitor is up-to 20% slower than Mutex, so if no block is given, use a mutex
+      @hash = hash
+    end
+
+    def synchronize(&blk)
+      if @mutex.owned?
+        yield(@hash)
+      else
+        @mutex.synchronize { yield(@hash) }
+      end
+    end
+
+    def method_missing(name, *args, &block)
+      if @hash.respond_to?(name)
+        @mutex.synchronize { @hash.public_send(name, *args, &block) }
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      @hash.respond_to?(name, include_private) || super
+    end
+  end
 
   class PendingQueue < Queue
     def shift_bucket
@@ -44,12 +81,12 @@ module Specwrk
     end
 
     def merge_with_previous_run_times!(h2)
-      h2.each { |_id, example| merge_example(example) }
+      synchronize do
+        h2.each { |_id, example| merge_example(example) }
 
-      # Sort by exepcted run time, slowest to fastest
-      new_h = sort_by { |_, example| example[:expected_run_time] }.reverse.to_h
-      clear
-      merge!(new_h)
+        # Sort by exepcted run time, slowest to fastest
+        @hash = @hash.sort_by { |_, example| example[:expected_run_time] }.reverse.to_h
+      end
     end
 
     private
@@ -67,15 +104,17 @@ module Specwrk
     def bucket_by_file
       bucket = []
 
-      key = keys.first
-      return [] if key.nil?
+      @mutex.synchronize do
+        key = @hash.keys.first
+        break if key.nil?
 
-      file_path = self[key][:file_path]
-      each do |id, example|
-        next unless example[:file_path] == file_path
+        file_path = @hash[key][:file_path]
+        @hash.each do |id, example|
+          next unless example[:file_path] == file_path
 
-        bucket << example
-        delete id
+          bucket << example
+          @hash.delete id
+        end
       end
 
       bucket
@@ -85,27 +124,30 @@ module Specwrk
     def bucket_by_timings
       bucket = []
 
-      estimated_run_time_total = 0
+      @mutex.synchronize do
+        estimated_run_time_total = 0
 
-      while estimated_run_time_total < run_time_bucket_threshold
-        key = keys.first
-        break if key.nil?
+        while estimated_run_time_total < run_time_bucket_threshold
+          key = @hash.keys.first
+          break if key.nil?
 
-        estimated_run_time_total += dig(key, :expected_run_time)
-        break if estimated_run_time_total > run_time_bucket_threshold && bucket.length.positive?
+          estimated_run_time_total += @hash.dig(key, :expected_run_time)
+          break if estimated_run_time_total > run_time_bucket_threshold && bucket.length.positive?
 
-        bucket << self[key]
-        delete key
+          bucket << @hash[key]
+          @hash.delete key
+        end
       end
 
       bucket
     end
 
+    # Ensure @mutex is held when calling this method
     def merge_example(example)
-      return if key? example[:id]
-      return if key? example[:file_path]
+      return if @hash.key? example[:id]
+      return if @hash.key? example[:file_path]
 
-      self[example[:id]] = if previous_run_times
+      @hash[example[:id]] = if previous_run_times
         example.merge!(
           expected_run_time: previous_run_times.dig(:examples, example[:id].to_sym, :run_time) || 99999.9 # run "unknown" files first
         )
@@ -123,24 +165,26 @@ module Specwrk
     end
 
     def dump
-      @run_times = []
-      @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
-      @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
+      @mutex.synchronize do
+        @run_times = []
+        @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
+        @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
 
-      @output = {
-        file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
-        meta: {failures: 0, passes: 0, pending: 0},
-        examples: {}
-      }
+        @output = {
+          file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
+          meta: {failures: 0, passes: 0, pending: 0},
+          examples: {}
+        }
 
-      values.each { |example| calculate(example) }
+        @hash.values.each { |example| calculate(example) }
 
-      @output[:meta][:total_run_time] = @run_times.sum
-      @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
-      @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
-      @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
+        @output[:meta][:total_run_time] = @run_times.sum
+        @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
+        @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
+        @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
 
-      @output
+        @output
+      end
     end
 
     private

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.7.1"
+  VERSION = "0.8.0"
 end

data/lib/specwrk/web/app.rb

@@ -21,6 +21,8 @@ require "specwrk/web/endpoints"
 module Specwrk
   class Web
     class App
+      REAP_INTERVAL = 330 # HTTP connection timeout + some buffer
+
       class << self
         def run!
           Process.setproctitle "specwrk-server"
@@ -72,6 +74,10 @@ module Specwrk
         end
       end
 
+      def initialize
+        @reaper_thread = Thread.new { reaper } unless ENV["SPECWRK_SRV_SINGLE_RUN"]
+      end
+
       def call(env)
         env[:request] ||= Rack::Request.new(env)
 
@@ -100,6 +106,26 @@ module Specwrk
           Endpoints::NotFound
         end
       end
+
+      def reaper
+        until Specwrk.force_quit
+          sleep REAP_INTERVAL
+
+          reap
+        end
+      end
+
+      def reap
+        Web::WORKERS.each do |run, workers|
+          most_recent_last_seen_at = workers.map { |id, worker| worker[:last_seen_at] }.max
+          next unless most_recent_last_seen_at
+
+          # Don't consider runs which aren't at least REAP_INTERVAL sec stale
+          if most_recent_last_seen_at < Time.now - REAP_INTERVAL
+            Web.clear_run_queues(run)
+          end
+        end
+      end
     end
   end
 end

data/lib/specwrk/web/auth.rb

@@ -28,7 +28,7 @@ module Specwrk
       private
 
       def unauthorized
-        [401, {"Content-Type" => "application/json"}, ["Unauthorized"]]
+        [401, {"content-type" => "application/json"}, ["Unauthorized"]]
       end
     end
   end

data/lib/specwrk/web/endpoints.rb

@@ -6,40 +6,14 @@ module Specwrk
   class Web
     module Endpoints
       class Base
-        attr_reader :pending_queue, :processing_queue, :completed_queue, :workers, :started_at
-
         def initialize(request)
           @request = request
-        end
 
-        def response
-          return with_response unless run_id # No run_id, no datastore usage in the endpoint
-
-          datastore.with_lock do |db|
-            @started_at = if db[:started_at]
-              Time.parse(db[:started_at])
-            else
-              db[:started_at] = Time.now
-            end
-
-            @pending_queue = PendingQueue.new.merge!(db[:pending] || {})
-            @processing_queue = Queue.new.merge!(db[:processing] || {})
-            @completed_queue = CompletedQueue.new.merge!(db[:completed] || {})
-            @workers = db[:workers] ||= {}
-
-            worker[:first_seen_at] ||= Time.now
-            worker[:last_seen_at] = Time.now
-
-            with_response.tap do
-              db[:pending] = pending_queue.to_h
-              db[:processing] = processing_queue.to_h
-              db[:completed] = completed_queue.to_h
-              db[:workers] = workers.to_h
-            end
-          end
+          worker[:first_seen_at] ||= Time.now
+          worker[:last_seen_at] = Time.now
         end
 
-        def with_response
+        def response
           not_found
         end
 
@@ -48,11 +22,11 @@ module Specwrk
         attr_reader :request
 
         def not_found
-          [404, {"Content-Type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
+          [404, {"content-type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
         end
 
         def ok
-          [200, {"Content-Type" => "text/plain"}, ["OK, 'ol chap"]]
+          [200, {"content-type" => "text/plain"}, ["OK, 'ol chap"]]
         end
 
         def payload
@@ -63,8 +37,24 @@ module Specwrk
           @body ||= request.body.read
         end
 
+        def pending_queue
+          Web::PENDING_QUEUES[run_id]
+        end
+
+        def processing_queue
+          Web::PROCESSING_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
+        end
+
+        def completed_queue
+          Web::COMPLETED_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
+        end
+
+        def workers
+          Web::WORKERS[request.get_header("HTTP_X_SPECWRK_RUN")]
+        end
+
         def worker
-          workers[request.get_header("HTTP_X_SPECWRK_ID")] ||= {}
+          workers[request.get_header("HTTP_X_SPECWRK_ID")]
         end
 
         def run_id
@@ -72,11 +62,7 @@ module Specwrk
         end
 
         def run_report_file_path
-          @run_report_file_path ||= File.join(ENV["SPECWRK_OUT"], run_id, "#{started_at.strftime("%Y%m%dT%H%M%S")}-report.json").to_s
-        end
-
-        def datastore
-          Web.datastore[File.join(ENV["SPECWRK_OUT"], run_id, "queues.json").to_s]
+          @run_report_file_path ||= File.join(ENV["SPECWRK_OUT"], "#{completed_queue.created_at.strftime("%Y%m%dT%H%M%S")}-report-#{run_id}.json").to_s
         end
       end
 
@@ -84,22 +70,27 @@ module Specwrk
       NotFound = Class.new(Base)
 
       class Health < Base
-        def with_response
+        def response
           [200, {}, []]
         end
       end
 
       class Heartbeat < Base
-        def with_response
+        def response
           ok
         end
       end
 
       class Seed < Base
-        def with_response
-          if ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"].nil? || pending_queue.length.zero?
-            examples = payload.map { |hash| [hash[:id], hash] }.to_h
-            pending_queue.merge_with_previous_run_times!(examples)
+        def response
+          pending_queue.synchronize do |pending_queue_hash|
+            unless ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"] && pending_queue_hash.length.positive?
+              examples = payload.map { |hash| [hash[:id], hash] }.to_h
+
+              pending_queue.merge_with_previous_run_times!(examples)
+
+              ok
+            end
           end
 
           ok
@@ -107,15 +98,16 @@ module Specwrk
       end
 
       class Complete < Base
-        def with_response
-          payload.each do |example|
-            next unless processing_queue.delete(example[:id].to_sym)
-            completed_queue[example[:id].to_sym] = example
+        def response
+          processing_queue.synchronize do |processing_queue_hash|
+            payload.each do |example|
+              next unless processing_queue_hash.delete(example[:id])
+              completed_queue[example[:id]] = example
+            end
           end
 
           if pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.positive? && ENV["SPECWRK_OUT"]
             completed_queue.dump_and_write(run_report_file_path)
-            FileUtils.ln_sf(run_report_file_path, File.join(ENV["SPECWRK_OUT"], "report.json"))
           end
 
           ok
@@ -123,19 +115,21 @@ module Specwrk
       end
 
       class Pop < Base
-        def with_response
-          @examples = pending_queue.shift_bucket
+        def response
+          processing_queue.synchronize do |processing_queue_hash|
+            @examples = pending_queue.shift_bucket
 
-          @examples.each do |example|
-            processing_queue[example[:id]] = example
+            @examples.each do |example|
+              processing_queue_hash[example[:id]] = example
+            end
           end
 
           if @examples.length.positive?
-            [200, {"Content-Type" => "application/json"}, [JSON.generate(@examples)]]
+            [200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
           elsif pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.zero?
-            [204, {"Content-Type" => "text/plain"}, ["Waiting for sample to be seeded."]]
+            [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
           elsif completed_queue.length.positive? && processing_queue.length.zero?
-            [410, {"Content-Type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
+            [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           else
             not_found
           end
@@ -143,11 +137,11 @@ module Specwrk
       end
 
       class Report < Base
-        def with_response
+        def response
           if data
-            [200, {"Content-Type" => "application/json"}, [data]]
+            [200, {"content-type" => "application/json"}, [data]]
           else
-            [404, {"Content-Type" => "text/plain"}, ["Unable to report on run #{run_id}; no file matching #{"*-report-#{run_id}.json"}"]]
+            [404, {"content-type" => "text/plain"}, ["Unable to report on run #{run_id}; no file matching #{"*-report-#{run_id}.json"}"]]
           end
         end
 
@@ -166,15 +160,15 @@ module Specwrk
         end
 
         def most_recent_run_report_file
-          @most_recent_run_report_file ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], run_id, "*-report.json")).last
+          @most_recent_run_report_file ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], "*-report-#{run_id}.json")).last
         end
       end
 
       class Shutdown < Base
-        def with_response
+        def response
           interupt! if ENV["SPECWRK_SRV_SINGLE_RUN"]
 
-          [200, {"Content-Type" => "text/plain"}, ["✌️"]]
+          [200, {"content-type" => "text/plain"}, ["✌️"]]
         end
 
         def interupt!

data/lib/specwrk/web.rb

@@ -1,13 +1,21 @@
 # frozen_string_literal: true
 
 require "specwrk/queue"
-require "specwrk/filestore"
 
 module Specwrk
   class Web
-    class << self
-      def datastore
-        Filestore
+    PENDING_QUEUES = Queue.new { |h, key| h[key] = PendingQueue.new }
+    PROCESSING_QUEUES = Queue.new { |h, key| h[key] = Queue.new }
+    COMPLETED_QUEUES = Queue.new { |h, key| h[key] = CompletedQueue.new }
+    WORKERS = Hash.new { |h, key| h[key] = Hash.new { |h, key| h[key] = {} } }
+
+    def self.clear_queues
+      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each(&:clear)
+    end
+
+    def self.clear_run_queues(run)
+      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each do |queue|
+        queue.delete(run)
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -152,6 +152,7 @@ files:
 - config.ru
 - docker/Dockerfile.server
 - docker/entrypoint.server.sh
+- docker/pitchfork.conf
 - examples/circleci/config.yml
 - examples/github/specwrk-multi-node.yml
 - examples/github/specwrk-single-node.yml
@@ -160,7 +161,6 @@ files:
 - lib/specwrk/cli.rb
 - lib/specwrk/cli_reporter.rb
 - lib/specwrk/client.rb
-- lib/specwrk/filestore.rb
 - lib/specwrk/hookable.rb
 - lib/specwrk/list_examples.rb
 - lib/specwrk/queue.rb

data/lib/specwrk/filestore.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-require "fileutils"
-
-module Specwrk
-  class Filestore
-    @mutexes = {}
-    @mutexes_mutex = Mutex.new # 🐢🐢🐢🐢
-
-    class << self
-      def [](path)
-        new(path)
-      end
-
-      def mutex_for(path)
-        @mutexes_mutex.synchronize do
-          @mutexes[path] ||= Mutex.new
-        end
-      end
-    end
-
-    def initialize(path)
-      @path = path
-      @tmpfile_path = @path + ".tmp"
-      @lock_path = @path + ".lock"
-
-      FileUtils.mkdir_p File.dirname(@path)
-      File.open(@path, "a") {} # multi-process and multi-thread safe touch
-    end
-
-    def with_lock
-      self.class.mutex_for(@path).synchronize do
-        lock_file.flock(File::LOCK_EX)
-
-        hash = read
-        result = yield(hash)
-
-        # Will truncate if already exists
-        File.open(@tmpfile_path, "w") do |tmpfile|
-          tmpfile.write(hash.to_json)
-          tmpfile.fsync
-          tmpfile.close
-        end
-
-        File.rename(@tmpfile_path, @path)
-        result
-      ensure
-        lock_file.flock(File::LOCK_UN)
-      end
-    end
-
-    private
-
-    def lock_file
-      @lock_file ||= File.open(@lock_path, "w")
-    end
-
-    def read
-      JSON.parse(File.read(@path), symbolize_names: true)
-    rescue JSON::ParserError
-      {}
-    end
-  end
-end