specwrk 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7467aa52809672311415ce6338aa0baa26b13cbb4fd4a2bf12854726f500517e
-  data.tar.gz: 1226cb23262a885106c39ec68c97d903c410976b12b4e857e739009284b42da2
+  metadata.gz: 4c6ab0d31757bbed9bf9f9ba02172a9bdd0b1ba57c7209f9d694b6471588ddd1
+  data.tar.gz: 44a634acd2023813f35b3eac9bcc331782c4cf624d716bf1d6341db80a07d464
 SHA512:
-  metadata.gz: c5ecdc6493127599390e7a95e13148cef715165c8ec4c3d897fc643eca7895e283dc21cb20f99b2f4d961dec8a4110c6e3ec3d077de0a609da6d2c58ee5a13a4
-  data.tar.gz: 39ec42792a73bd68b693b81b379e54bb959fe21a78bfe811f645135e11e28793e5daf23b1f39a6d36fe81b56ae46963b4cdf95df632230adcc09c71460fae92f
+  metadata.gz: 6c29b9f3e52b7b4d88920b6a4d92cba4b24fed583573b9cfa8dd273777a73534ded97f2a89ecc078d58d183e48a0095a1403900de7c3d497954f6c9002b4e962
+  data.tar.gz: ce81b65408313ce7088ba157c4578ffae66c15f9f8d4e053992c8eb1c4c99a9ecbc3cb68e1903e6817d988740b7dd57e282866a06a23146841ca057adb89fb19

data/docker/Dockerfile.server

@@ -1,6 +1,11 @@
 FROM ruby:3.4-alpine
 
-RUN apk add --no-cache build-base 
+RUN apk add --no-cache \
+      build-base \
+      ruby-dev \
+      linux-headers \
+      zlib-dev \
+      libffi-dev
 
 WORKDIR /app
 
@@ -8,15 +13,14 @@ RUN mkdir .specwrk/
 
 ARG SPECWRK_SRV_PORT=5138
 ARG SPECWRK_VERSION=latest
-ARG GEM_FILE=specwrk-$SPECWRK_VERSION.gem
+ARG GEMFILE=specwrk-$SPECWRK_VERSION.gem
 
-COPY $GEM_FILE ./
-RUN gem install ./$GEM_FILE --no-document
-RUN rm ./$GEM_FILE
+COPY $GEMFILE ./
+RUN gem install ./$GEMFILE --no-document
+RUN rm ./$GEMFILE
 
-RUN gem install pitchfork thruster
+RUN gem install puma thruster
 COPY config.ru ./
-COPY docker/pitchfork.conf ./
 
 COPY docker/entrypoint.server.sh /usr/local/bin/entrypoint
 RUN chmod +x /usr/local/bin/entrypoint

data/docker/entrypoint.server.sh

@@ -2,6 +2,6 @@
 
 export THRUSTER_HTTP_PORT=${PORT:-5138}
 export THRUSTER_TARGET_PORT=3000
-export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-300}
+export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-305}
 
-exec thrust pitchfork -c pitchfork.conf
+exec thrust puma --workers 0 --bind tcp://127.0.0.1:3000 --threads ${PUMA_THREADS:-1}

data/examples/circleci/config.yml

@@ -0,0 +1 @@
+.circleci/config.yml

data/examples/github/specwrk-multi-node.yml

@@ -0,0 +1 @@
+.github/workflows/specwrk-multi-node.yml

data/examples/github/specwrk-single-node.yml

@@ -0,0 +1 @@
+.github/workflows/specwrk-single-node.yml

data/exe/specwrk

@@ -9,6 +9,7 @@ trap("INT") do
 
     Specwrk.force_quit = true
   elsif Specwrk.starting_pid != Process.pid
+    RSpec.world.wants_to_quit = true if defined?(RSpec)
     exit(1) if Specwrk.force_quit
     Specwrk.force_quit = true
   end

data/lib/specwrk/filestore.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Specwrk
+  class Filestore
+    @mutexes = {}
+    @mutexes_mutex = Mutex.new # 🐢🐢🐢🐢
+
+    class << self
+      def [](path)
+        new(path)
+      end
+
+      def mutex_for(path)
+        @mutexes_mutex.synchronize do
+          @mutexes[path] ||= Mutex.new
+        end
+      end
+    end
+
+    def initialize(path)
+      @path = path
+      @tmpfile_path = @path + ".tmp"
+      @lock_path = @path + ".lock"
+
+      FileUtils.mkdir_p File.dirname(@path)
+      File.open(@path, "a") {} # multi-process and multi-thread safe touch
+    end
+
+    def with_lock
+      self.class.mutex_for(@path).synchronize do
+        lock_file.flock(File::LOCK_EX)
+
+        hash = read
+        result = yield(hash)
+
+        # Will truncate if already exists
+        File.open(@tmpfile_path, "w") do |tmpfile|
+          tmpfile.write(hash.to_json)
+          tmpfile.fsync
+          tmpfile.close
+        end
+
+        File.rename(@tmpfile_path, @path)
+        result
+      ensure
+        lock_file.flock(File::LOCK_UN)
+      end
+    end
+
+    private
+
+    def lock_file
+      @lock_file ||= File.open(@lock_path, "w")
+    end
+
+    def read
+      JSON.parse(File.read(@path), symbolize_names: true)
+    rescue JSON::ParserError
+      {}
+    end
+  end
+end

data/lib/specwrk/queue.rb

@@ -4,44 +4,7 @@ require "time"
 require "json"
 
 module Specwrk
-  # Thread-safe Hash access
-  class Queue
-    attr_reader :created_at
-
-    def initialize(hash = {})
-      @created_at = Time.now
-
-      if block_given?
-        @mutex = Monitor.new # Reentrant locking is required here
-        # It's possible to enter the proc from two threads, so we need to ||= in case
-        # one thread has set a value prior to the yield.
-        hash.default_proc = proc { |h, key| @mutex.synchronize { yield(h, key) } }
-      end
-
-      @mutex ||= Mutex.new # Monitor is up-to 20% slower than Mutex, so if no block is given, use a mutex
-      @hash = hash
-    end
-
-    def synchronize(&blk)
-      if @mutex.owned?
-        yield(@hash)
-      else
-        @mutex.synchronize { yield(@hash) }
-      end
-    end
-
-    def method_missing(name, *args, &block)
-      if @hash.respond_to?(name)
-        @mutex.synchronize { @hash.public_send(name, *args, &block) }
-      else
-        super
-      end
-    end
-
-    def respond_to_missing?(name, include_private = false)
-      @hash.respond_to?(name, include_private) || super
-    end
-  end
+  Queue = Class.new(Hash)
 
   class PendingQueue < Queue
     def shift_bucket
@@ -81,12 +44,12 @@ module Specwrk
     end
 
     def merge_with_previous_run_times!(h2)
-      synchronize do
-        h2.each { |_id, example| merge_example(example) }
+      h2.each { |_id, example| merge_example(example) }
 
-        # Sort by exepcted run time, slowest to fastest
-        @hash = @hash.sort_by { |_, example| example[:expected_run_time] }.reverse.to_h
-      end
+      # Sort by exepcted run time, slowest to fastest
+      new_h = sort_by { |_, example| example[:expected_run_time] }.reverse.to_h
+      clear
+      merge!(new_h)
     end
 
     private
@@ -104,17 +67,15 @@ module Specwrk
     def bucket_by_file
       bucket = []
 
-      @mutex.synchronize do
-        key = @hash.keys.first
-        break if key.nil?
+      key = keys.first
+      return [] if key.nil?
 
-        file_path = @hash[key][:file_path]
-        @hash.each do |id, example|
-          next unless example[:file_path] == file_path
+      file_path = self[key][:file_path]
+      each do |id, example|
+        next unless example[:file_path] == file_path
 
-          bucket << example
-          @hash.delete id
-        end
+        bucket << example
+        delete id
       end
 
       bucket
@@ -124,30 +85,27 @@ module Specwrk
     def bucket_by_timings
       bucket = []
 
-      @mutex.synchronize do
-        estimated_run_time_total = 0
+      estimated_run_time_total = 0
 
-        while estimated_run_time_total < run_time_bucket_threshold
-          key = @hash.keys.first
-          break if key.nil?
+      while estimated_run_time_total < run_time_bucket_threshold
+        key = keys.first
+        break if key.nil?
 
-          estimated_run_time_total += @hash.dig(key, :expected_run_time)
-          break if estimated_run_time_total > run_time_bucket_threshold && bucket.length.positive?
+        estimated_run_time_total += dig(key, :expected_run_time)
+        break if estimated_run_time_total > run_time_bucket_threshold && bucket.length.positive?
 
-          bucket << @hash[key]
-          @hash.delete key
-        end
+        bucket << self[key]
+        delete key
       end
 
       bucket
     end
 
-    # Ensure @mutex is held when calling this method
     def merge_example(example)
-      return if @hash.key? example[:id]
-      return if @hash.key? example[:file_path]
+      return if key? example[:id]
+      return if key? example[:file_path]
 
-      @hash[example[:id]] = if previous_run_times
+      self[example[:id]] = if previous_run_times
         example.merge!(
           expected_run_time: previous_run_times.dig(:examples, example[:id].to_sym, :run_time) || 99999.9 # run "unknown" files first
         )
@@ -165,26 +123,24 @@ module Specwrk
     end
 
     def dump
-      @mutex.synchronize do
-        @run_times = []
-        @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
-        @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
+      @run_times = []
+      @first_started_at = Time.new(2999, 1, 1, 0, 0, 0) # TODO: Make future proof /s
+      @last_finished_at = Time.new(1900, 1, 1, 0, 0, 0)
 
-        @output = {
-          file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
-          meta: {failures: 0, passes: 0, pending: 0},
-          examples: {}
-        }
+      @output = {
+        file_totals: Hash.new { |h, filename| h[filename] = 0.0 },
+        meta: {failures: 0, passes: 0, pending: 0},
+        examples: {}
+      }
 
-        @hash.values.each { |example| calculate(example) }
+      values.each { |example| calculate(example) }
 
-        @output[:meta][:total_run_time] = @run_times.sum
-        @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
-        @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
-        @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
+      @output[:meta][:total_run_time] = @run_times.sum
+      @output[:meta][:average_run_time] = @output[:meta][:total_run_time] / [@run_times.length, 1].max.to_f
+      @output[:meta][:first_started_at] = @first_started_at.iso8601(6)
+      @output[:meta][:last_finished_at] = @last_finished_at.iso8601(6)
 
-        @output
-      end
+      @output
     end
 
     private

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.6.2"
+  VERSION = "0.7.0"
 end

data/lib/specwrk/web/app.rb

@@ -74,10 +74,6 @@ module Specwrk
         end
       end
 
-      def initialize
-        @reaper_thread = Thread.new { reaper } unless ENV["SPECWRK_SRV_SINGLE_RUN"]
-      end
-
       def call(env)
         env[:request] ||= Rack::Request.new(env)
 
@@ -106,26 +102,6 @@ module Specwrk
           Endpoints::NotFound
         end
       end
-
-      def reaper
-        until Specwrk.force_quit
-          sleep REAP_INTERVAL
-
-          reap
-        end
-      end
-
-      def reap
-        Web::WORKERS.each do |run, workers|
-          most_recent_last_seen_at = workers.map { |id, worker| worker[:last_seen_at] }.max
-          next unless most_recent_last_seen_at
-
-          # Don't consider runs which aren't at least REAP_INTERVAL sec stale
-          if most_recent_last_seen_at < Time.now - REAP_INTERVAL
-            Web.clear_run_queues(run)
-          end
-        end
-      end
     end
   end
 end

data/lib/specwrk/web/auth.rb

@@ -28,7 +28,7 @@ module Specwrk
       private
 
       def unauthorized
-        [401, {"content-type" => "application/json"}, ["Unauthorized"]]
+        [401, {"Content-Type" => "application/json"}, ["Unauthorized"]]
       end
     end
   end

data/lib/specwrk/web/endpoints.rb

@@ -6,14 +6,38 @@ module Specwrk
   class Web
     module Endpoints
       class Base
+        attr_reader :pending_queue, :processing_queue, :completed_queue, :workers, :started_at
+
         def initialize(request)
           @request = request
-
-          worker[:first_seen_at] ||= Time.now
-          worker[:last_seen_at] = Time.now
         end
 
         def response
+          datastore.with_lock do |db|
+            @started_at = if db[:started_at]
+              Time.parse(db[:started_at])
+            else
+              db[:started_at] = Time.now
+            end
+
+            @pending_queue = PendingQueue.new.merge!(db[:pending] || {})
+            @processing_queue = Queue.new.merge!(db[:processing] || {})
+            @completed_queue = CompletedQueue.new.merge!(db[:completed] || {})
+            @workers = db[:workers] ||= {}
+
+            worker[:first_seen_at] ||= Time.now
+            worker[:last_seen_at] = Time.now
+
+            with_response.tap do
+              db[:pending] = pending_queue.to_h
+              db[:processing] = processing_queue.to_h
+              db[:completed] = completed_queue.to_h
+              db[:workers] = workers.to_h
+            end
+          end
+        end
+
+        def with_response
           not_found
         end
 
@@ -22,11 +46,11 @@ module Specwrk
         attr_reader :request
 
         def not_found
-          [404, {"content-type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
+          [404, {"Content-Type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
         end
 
         def ok
-          [200, {"content-type" => "text/plain"}, ["OK, 'ol chap"]]
+          [200, {"Content-Type" => "text/plain"}, ["OK, 'ol chap"]]
         end
 
         def payload
@@ -37,24 +61,8 @@ module Specwrk
           @body ||= request.body.read
         end
 
-        def pending_queue
-          Web::PENDING_QUEUES[run_id]
-        end
-
-        def processing_queue
-          Web::PROCESSING_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
-        end
-
-        def completed_queue
-          Web::COMPLETED_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
-        end
-
-        def workers
-          Web::WORKERS[request.get_header("HTTP_X_SPECWRK_RUN")]
-        end
-
         def worker
-          workers[request.get_header("HTTP_X_SPECWRK_ID")]
+          workers[request.get_header("HTTP_X_SPECWRK_ID")] ||= {}
         end
 
         def run_id
@@ -62,7 +70,11 @@ module Specwrk
         end
 
         def run_report_file_path
-          @run_report_file_path ||= File.join(ENV["SPECWRK_OUT"], "#{completed_queue.created_at.strftime("%Y%m%dT%H%M%S")}-report-#{run_id}.json").to_s
+          @run_report_file_path ||= File.join(ENV["SPECWRK_OUT"], run_id, "#{started_at.strftime("%Y%m%dT%H%M%S")}-report.json").to_s
+        end
+
+        def datastore
+          Web.datastore[File.join(ENV["SPECWRK_OUT"], run_id, "queues.json").to_s]
         end
       end
 
@@ -70,27 +82,22 @@ module Specwrk
       NotFound = Class.new(Base)
 
       class Health < Base
-        def response
+        def with_response
           [200, {}, []]
         end
       end
 
       class Heartbeat < Base
-        def response
+        def with_response
           ok
         end
       end
 
       class Seed < Base
-        def response
-          pending_queue.synchronize do |pending_queue_hash|
-            unless ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"] && pending_queue_hash.length.positive?
-              examples = payload.map { |hash| [hash[:id], hash] }.to_h
-
-              pending_queue.merge_with_previous_run_times!(examples)
-
-              ok
-            end
+        def with_response
+          if ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"].nil? || pending_queue.length.zero?
+            examples = payload.map { |hash| [hash[:id], hash] }.to_h
+            pending_queue.merge_with_previous_run_times!(examples)
           end
 
           ok
@@ -98,16 +105,15 @@ module Specwrk
       end
 
       class Complete < Base
-        def response
-          processing_queue.synchronize do |processing_queue_hash|
-            payload.each do |example|
-              next unless processing_queue_hash.delete(example[:id])
-              completed_queue[example[:id]] = example
-            end
+        def with_response
+          payload.each do |example|
+            next unless processing_queue.delete(example[:id].to_sym)
+            completed_queue[example[:id].to_sym] = example
           end
 
           if pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.positive? && ENV["SPECWRK_OUT"]
             completed_queue.dump_and_write(run_report_file_path)
+            FileUtils.ln_sf(run_report_file_path, File.join(ENV["SPECWRK_OUT"], "report.json"))
           end
 
           ok
@@ -115,21 +121,19 @@ module Specwrk
       end
 
       class Pop < Base
-        def response
-          processing_queue.synchronize do |processing_queue_hash|
-            @examples = pending_queue.shift_bucket
+        def with_response
+          @examples = pending_queue.shift_bucket
 
-            @examples.each do |example|
-              processing_queue_hash[example[:id]] = example
-            end
+          @examples.each do |example|
+            processing_queue[example[:id]] = example
           end
 
           if @examples.length.positive?
-            [200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
+            [200, {"Content-Type" => "application/json"}, [JSON.generate(@examples)]]
           elsif pending_queue.length.zero? && processing_queue.length.zero? && completed_queue.length.zero?
-            [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
+            [204, {"Content-Type" => "text/plain"}, ["Waiting for sample to be seeded."]]
           elsif completed_queue.length.positive? && processing_queue.length.zero?
-            [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
+            [410, {"Content-Type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           else
             not_found
           end
@@ -137,11 +141,11 @@ module Specwrk
       end
 
       class Report < Base
-        def response
+        def with_response
           if data
-            [200, {"content-type" => "application/json"}, [data]]
+            [200, {"Content-Type" => "application/json"}, [data]]
           else
-            [404, {"content-type" => "text/plain"}, ["Unable to report on run #{run_id}; no file matching #{"*-report-#{run_id}.json"}"]]
+            [404, {"Content-Type" => "text/plain"}, ["Unable to report on run #{run_id}; no file matching #{"*-report-#{run_id}.json"}"]]
           end
         end
 
@@ -160,15 +164,15 @@ module Specwrk
         end
 
         def most_recent_run_report_file
-          @most_recent_run_report_file ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], "*-report-#{run_id}.json")).last
+          @most_recent_run_report_file ||= Dir.glob(File.join(ENV["SPECWRK_OUT"], run_id, "*-report.json")).last
         end
       end
 
       class Shutdown < Base
-        def response
+        def with_response
           interupt! if ENV["SPECWRK_SRV_SINGLE_RUN"]
 
-          [200, {"content-type" => "text/plain"}, ["✌️"]]
+          [200, {"Content-Type" => "text/plain"}, ["✌️"]]
         end
 
         def interupt!

data/lib/specwrk/web.rb

@@ -1,21 +1,13 @@
 # frozen_string_literal: true
 
 require "specwrk/queue"
+require "specwrk/filestore"
 
 module Specwrk
   class Web
-    PENDING_QUEUES = Queue.new { |h, key| h[key] = PendingQueue.new }
-    PROCESSING_QUEUES = Queue.new { |h, key| h[key] = Queue.new }
-    COMPLETED_QUEUES = Queue.new { |h, key| h[key] = CompletedQueue.new }
-    WORKERS = Hash.new { |h, key| h[key] = Hash.new { |h, key| h[key] = {} } }
-
-    def self.clear_queues
-      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each(&:clear)
-    end
-
-    def self.clear_run_queues(run)
-      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each do |queue|
-        queue.delete(run)
+    class << self
+      def datastore
+        Filestore
       end
     end
   end

data/lib/specwrk/worker.rb

@@ -19,6 +19,7 @@ module Specwrk
       @running = true
       @client = Client.new
       @executor = Executor.new
+      @all_examples_completed = false
       @seed_waits = ENV.fetch("SPECWRK_SEED_WAITS", "10").to_i
       @heartbeat_thread ||= Thread.new do
         thump
@@ -33,6 +34,7 @@ module Specwrk
 
         execute
       rescue CompletedAllExamplesError
+        @all_examples_completed = true
         break
       rescue NoMoreExamplesError
         # Wait for the other processes (workers) on the same host to finish
@@ -102,7 +104,7 @@ module Specwrk
     attr_reader :running, :client, :executor
 
     def status
-      return 1 unless executor.example_processed
+      return 1 if !executor.example_processed && !@all_examples_completed
       return 1 if executor.failure
       return 1 if Specwrk.force_quit
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -152,12 +152,15 @@ files:
 - config.ru
 - docker/Dockerfile.server
 - docker/entrypoint.server.sh
-- docker/pitchfork.conf
+- examples/circleci/config.yml
+- examples/github/specwrk-multi-node.yml
+- examples/github/specwrk-single-node.yml
 - exe/specwrk
 - lib/specwrk.rb
 - lib/specwrk/cli.rb
 - lib/specwrk/cli_reporter.rb
 - lib/specwrk/client.rb
+- lib/specwrk/filestore.rb
 - lib/specwrk/hookable.rb
 - lib/specwrk/list_examples.rb
 - lib/specwrk/queue.rb

data/docker/pitchfork.conf

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-worker_processes 1
-listen "localhost:3000", backlog: 2048
-timeout 301