specwrk 0.9.1 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 657cd8e611ea7ea6344dc5bd540892e69d801ea5afd410b1be2e7a6d3f73658e
-  data.tar.gz: c066775a6f57452cd55c49eec7604b8e264881ad085d7847ff176240a9ecbe05
+  metadata.gz: 9dfd4ef167bef2dc36de450f5855ff2ff376255ddd1e471426e08cb20d91a615
+  data.tar.gz: 0107c130003c627c5f3cc50949d9dc6cd3563e4f810b799527e6aa2234a19822
 SHA512:
-  metadata.gz: 076a41a98119ccfe0108c76c32341d8cbf6642da3f37f80276a61ed885fbca9ddaf31c597b8462ddeb608c67d7e8787f5cefd606cb4b977aeef715e9a5cdb0bb
-  data.tar.gz: 39891db9a0400196ddaba587e7014aa00e485c65ce37efc997998f854dd98d8cabfef39806657bec15e71c43a29743dda4081b35c2022a917a5503e4e161cea4
+  metadata.gz: 1ffb03334f25bb2ca19a45e3004feb80ab3565e12c2e8aad43565000e741b37b2aa296683c1cf5215571cd25a79f5f53934cdfc427fea89a44e92d49d8c0a076
+  data.tar.gz: 4eb0f0e1a9469b1d0f23952d06b24b873d98e25ec3777bedf94a0b2be904038b225f39e9395556350c8c1cd057086f7af8866fbb38bf79ea5937b531be690993

data/README.md

@@ -58,7 +58,6 @@ Options:
   --port=VALUE, -p VALUE            # Server port. Overrides SPECWRK_SRV_PORT, default: "5138"
   --bind=VALUE, -b VALUE            # Server bind address. Overrides SPECWRK_SRV_BIND, default: "127.0.0.1"
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]single-seed-per-run        # Only allow one seed per run. Useful for CI where many nodes may seed at the same time, default: false
   --[no-]verbose                    # Run in verbose mode. Default false., default: false
   --help, -h                        # Print this help
 ```
@@ -83,7 +82,6 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]single-seed-per-run        # Only allow one seed per run. Useful for CI where many nodes may seed at the same time, default: false
   --[no-]verbose                    # Run in verbose mode. Default false., default: false
   --[no-]single-run                 # Act on shutdown requests from clients. Default: false., default: false
   --help, -h                        # Print this help

data/docker/Dockerfile.server

@@ -1,11 +1,6 @@
 FROM ruby:3.4-alpine
 
-RUN apk add --no-cache \
-      build-base \
-      ruby-dev \
-      linux-headers \
-      zlib-dev \
-      libffi-dev
+RUN apk add --no-cache build-base 
 
 WORKDIR /app
 
@@ -13,14 +8,15 @@ RUN mkdir .specwrk/
 
 ARG SPECWRK_SRV_PORT=5138
 ARG SPECWRK_VERSION=latest
-ARG GEMFILE=specwrk-$SPECWRK_VERSION.gem
+ARG GEM_FILE=specwrk-$SPECWRK_VERSION.gem
 
-COPY $GEMFILE ./
-RUN gem install ./$GEMFILE --no-document
-RUN rm ./$GEMFILE
+COPY $GEM_FILE ./
+RUN gem install ./$GEM_FILE --no-document
+RUN rm ./$GEM_FILE
 
-RUN gem install puma thruster
+RUN gem install pitchfork thruster
 COPY config.ru ./
+COPY docker/pitchfork.conf ./
 
 COPY docker/entrypoint.server.sh /usr/local/bin/entrypoint
 RUN chmod +x /usr/local/bin/entrypoint

data/docker/entrypoint.server.sh

@@ -2,6 +2,6 @@
 
 export THRUSTER_HTTP_PORT=${PORT:-5138}
 export THRUSTER_TARGET_PORT=3000
-export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-305}
+export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-300}
 
-exec thrust puma --workers 0 --bind tcp://127.0.0.1:3000 --threads ${PUMA_THREADS:-1} --workers ${PUMA_WORKERS:-0}
+exec thrust pitchfork -c pitchfork.conf

data/docker/pitchfork.conf

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+worker_processes 1
+listen "localhost:3000", backlog: 2048
+timeout 301

data/lib/specwrk/cli.rb

@@ -76,7 +76,6 @@ module Specwrk
         base.unique_option :key, type: :string, aliases: ["-k"], default: ENV.fetch("SPECWRK_SRV_KEY", ""), desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
         base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker output is stored. Overrides SPECWRK_OUT"
         base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWERK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY"
-        base.unique_option :single_seed_per_run, type: :boolean, default: false, desc: "Only allow one seed per run. Useful for CI where many nodes may seed at the same time"
         base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode. Default false."
       end
 
@@ -211,7 +210,8 @@ module Specwrk
           status "Server responding ✓"
           status "Seeding #{examples.length} examples..."
           Client.new.seed(examples)
-          status "Samples seeded ✓"
+          file_count = examples.group_by { |e| e[:file_path] }.keys.size
+          status "🌱 Seeded #{examples.size} examples across #{file_count} files"
         end
 
         if Specwrk.wait_for_pids_exit([seed_pid]).value?(1)

data/lib/specwrk/client.rb

@@ -105,6 +105,23 @@ module Specwrk
       (response.code == "200") ? true : raise(UnhandledResponseError.new("#{response.code}: #{response.body}"))
     end
 
+    def complete_and_fetch_examples(examples)
+      response = post "/complete_and_pop", body: examples.to_json
+
+      case response.code
+      when "200"
+        JSON.parse(response.body, symbolize_names: true)
+      when "204"
+        raise WaitingForSeedError
+      when "404"
+        raise NoMoreExamplesError
+      when "410"
+        raise CompletedAllExamplesError
+      else
+        raise UnhandledResponseError.new("#{response.code}: #{response.body}")
+      end
+    end
+
     def seed(examples)
       response = post "/seed", body: examples.to_json
 

data/lib/specwrk/store.rb

@@ -7,40 +7,24 @@ require "specwrk/store/file_adapter"
 
 module Specwrk
   class Store
-    MUTEXES = {}
-    MUTEXES_MUTEX = Mutex.new # 🐢🐢🐢🐢
-
-    class << self
-      def mutex_for(path)
-        MUTEXES_MUTEX.synchronize do
-          MUTEXES[path] ||= Mutex.new
-        end
-      end
-    end
-
-    def initialize(path, thread_safe_reads: true)
+    def initialize(path)
       @path = path
-      @thread_safe_reads = thread_safe_reads
     end
 
     def [](key)
-      sync(thread_safe: thread_safe_reads) { adapter[key.to_s] }
+      adapter[key.to_s]
     end
 
     def multi_read(*keys)
-      sync(thread_safe: thread_safe_reads) { adapter.multi_read(*keys) }
+      adapter.multi_read(*keys)
     end
 
     def []=(key, value)
-      sync do
-        adapter[key.to_s] = value
-      end
+      adapter[key.to_s] = value
     end
 
     def keys
-      all_keys = sync(thread_safe: thread_safe_reads) do
-        adapter.keys
-      end
+      all_keys = adapter.keys
 
       all_keys.reject { |k| k.start_with? "____" }
     end
@@ -54,28 +38,24 @@ module Specwrk
     end
 
     def empty?
-      sync(thread_safe: thread_safe_reads) do
-        adapter.empty?
-      end
+      adapter.empty?
     end
 
     def delete(*keys)
-      sync { adapter.delete(*keys) }
+      adapter.delete(*keys)
     end
 
     def merge!(h2)
       h2.transform_keys!(&:to_s)
-      sync { adapter.merge!(h2) }
+      adapter.merge!(h2)
     end
 
     def clear
-      sync { adapter.clear }
+      adapter.clear
     end
 
     def to_h
-      sync(thread_safe: thread_safe_reads) do
-        adapter.multi_read(*keys).transform_keys!(&:to_sym)
-      end
+      adapter.multi_read(*keys).transform_keys!(&:to_sym)
     end
 
     def inspect
@@ -92,16 +72,6 @@ module Specwrk
 
     private
 
-    attr_reader :thread_safe_reads
-
-    def sync(thread_safe: true)
-      if !thread_safe || mutex.owned?
-        yield
-      else
-        mutex.synchronize { yield }
-      end
-    end
-
     def adapter
       @adapter ||= FileAdapter.new(@path)
     end
@@ -123,15 +93,13 @@ module Specwrk
     end
 
     def shift_bucket
-      sync do
-        return bucket_by_file unless run_time_bucket_maximum&.positive?
-
-        case ENV["SPECWRK_SRV_GROUP_BY"]
-        when "file"
-          bucket_by_file
-        else
-          bucket_by_timings
-        end
+      return bucket_by_file unless run_time_bucket_maximum&.positive?
+
+      case ENV["SPECWRK_SRV_GROUP_BY"]
+      when "file"
+        bucket_by_file
+      else
+        bucket_by_timings
       end
     end
 

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.9.1"
+  VERSION = "0.10.1"
 end

data/lib/specwrk/web/app.rb

@@ -90,6 +90,8 @@ module Specwrk
           Endpoints::Pop
         when ["POST", "/complete"]
           Endpoints::Complete
+        when ["POST", "/complete_and_pop"]
+          Endpoints::CompleteAndPop
         when ["POST", "/seed"]
           Endpoints::Seed
         when ["GET", "/report"]

data/lib/specwrk/web/auth.rb

@@ -28,7 +28,7 @@ module Specwrk
       private
 
       def unauthorized
-        [401, {"Content-Type" => "application/json"}, ["Unauthorized"]]
+        [401, {"content-type" => "application/json"}, ["Unauthorized"]]
       end
     end
   end

data/lib/specwrk/web/endpoints.rb

@@ -51,11 +51,11 @@ module Specwrk
         end
 
         def not_found
-          [404, {"Content-Type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
+          [404, {"content-type" => "text/plain"}, ["This is not the path you're looking for, 'ol chap..."]]
         end
 
         def ok
-          [200, {"Content-Type" => "text/plain"}, ["OK, 'ol chap"]]
+          [200, {"content-type" => "text/plain"}, ["OK, 'ol chap"]]
         end
 
         def payload
@@ -83,11 +83,11 @@ module Specwrk
         end
 
         def metadata
-          @metadata ||= Store.new(File.join(datastore_path, "metadata"), thread_safe_reads: false)
+          @metadata ||= Store.new(File.join(datastore_path, "metadata"))
         end
 
         def run_times
-          @run_times ||= Store.new(File.join(ENV["SPECWRK_OUT"], "run_times"), thread_safe_reads: false)
+          @run_times ||= Store.new(File.join(ENV["SPECWRK_OUT"], "run_times"))
         end
 
         def worker
@@ -137,17 +137,15 @@ module Specwrk
 
       class Seed < Base
         def before_lock
-          examples_with_run_times if persist_seeds?
+          examples_with_run_times
         end
 
         def with_response
-          if persist_seeds?
-            new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
-            pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
-
-            pending.merge!(examples_with_run_times)
-          end
+          pending.clear
+          new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
+          pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
 
+          pending.merge!(examples_with_run_times)
           processing.clear
           completed.clear
 
@@ -192,10 +190,6 @@ module Specwrk
           (mean + Math.sqrt(variance)).round(2)
         end
 
-        def persist_seeds?
-          ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"].nil? || pending.empty?
-        end
-
         def sort_by
           if ENV["SPECWRK_SRV_GROUP_BY"] == "file" || run_times.empty?
             :file
@@ -207,6 +201,7 @@ module Specwrk
 
       class Complete < Base
         def with_response
+          warn "[DEPRECATED] This endpoint will be retired in favor of CompleteAndPop. Upgrade your clients."
           completed.merge!(completed_examples)
           processing.delete(*completed_examples.keys)
 
@@ -235,20 +230,65 @@ module Specwrk
           processing.merge!(processing_data)
 
           if @examples.any?
-            [200, {"Content-Type" => "application/json"}, [JSON.generate(@examples)]]
+            [200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
+          elsif pending.empty? && processing.empty? && completed.empty?
+            [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
+          elsif completed.any? && processing.empty?
+            [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
+          else
+            not_found
+          end
+        end
+      end
+
+      class CompleteAndPop < Base
+        def with_response
+          completed.merge!(completed_examples)
+          run_times.merge! run_time_data
+          processing.delete(*completed_examples.keys)
+
+          @examples = pending.shift_bucket
+
+          processing_data = @examples.map { |example| [example[:id], example] }.to_h
+          processing.merge!(processing_data)
+
+          if @examples.any?
+            [200, {"content-type" => "application/json"}, [JSON.generate(@examples)]]
           elsif pending.empty? && processing.empty? && completed.empty?
-            [204, {"Content-Type" => "text/plain"}, ["Waiting for sample to be seeded."]]
+            [204, {"content-type" => "text/plain"}, ["Waiting for sample to be seeded."]]
           elsif completed.any? && processing.empty?
-            [410, {"Content-Type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
+            [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           else
             not_found
           end
         end
+
+        private
+
+        def before_lock
+          completed_examples
+          run_time_data
+        end
+
+        def completed_examples
+          @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
+        end
+
+        # We don't care about exact values here, just approximate run times are fine
+        # So if we overwrite run times from another process it is nbd
+        def after_lock
+          # run_time_data = payload.map { |example| [example[:id], example[:run_time]] }.to_h
+          # run_times.merge! run_time_data
+        end
+
+        def run_time_data
+          @run_time_data ||= payload.map { |example| [example[:id], example[:run_time]] }.to_h
+        end
       end
 
       class Report < Base
         def with_response
-          [200, {"Content-Type" => "application/json"}, [JSON.generate(completed.dump)]]
+          [200, {"content-type" => "application/json"}, [JSON.generate(completed.dump)]]
         end
       end
 
@@ -259,7 +299,7 @@ module Specwrk
 
           interupt! if ENV["SPECWRK_SRV_SINGLE_RUN"]
 
-          [200, {"Content-Type" => "text/plain"}, ["✌️"]]
+          [200, {"content-type" => "text/plain"}, ["✌️"]]
         end
 
         def interupt!

data/lib/specwrk/worker.rb

@@ -69,15 +69,21 @@ module Specwrk
     end
 
     def execute
-      executor.run client.fetch_examples
+      executor.run next_examples
       complete_examples
     rescue UnhandledResponseError => e
-      # If fetching examples fails we can just try again so warn and return
+      # If fetching examples via next_exampels fails we can just try again so warn and return
+      # Expects complete_examples to rescue this error if raised in that method
       warn e.message
     end
 
+    def next_examples
+      return @next_examples if @next_examples&.length&.positive?
+      client.fetch_examples
+    end
+
     def complete_examples
-      client.complete_examples executor.examples
+      @next_examples = client.complete_and_fetch_examples executor.examples
     rescue UnhandledResponseError => e
       # I do not think we should so lightly abandon the completion of executed examples
       # try to complete until successful or terminated

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.10.1
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -180,6 +180,7 @@ files:
 - config.ru
 - docker/Dockerfile.server
 - docker/entrypoint.server.sh
+- docker/pitchfork.conf
 - examples/circleci/config.yml
 - examples/github/specwrk-multi-node.yml
 - examples/github/specwrk-single-node.yml