specwrk 0.9.1 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 657cd8e611ea7ea6344dc5bd540892e69d801ea5afd410b1be2e7a6d3f73658e
-  data.tar.gz: c066775a6f57452cd55c49eec7604b8e264881ad085d7847ff176240a9ecbe05
+  metadata.gz: 656884cedc1e6851a42054d696fb0c8cbb3c7afffa48935340c973d824cc6cc0
+  data.tar.gz: e8b562b8070e5342d6343b299a80fe0bc98a17060632aa369f5ded91ffecb820
 SHA512:
-  metadata.gz: 076a41a98119ccfe0108c76c32341d8cbf6642da3f37f80276a61ed885fbca9ddaf31c597b8462ddeb608c67d7e8787f5cefd606cb4b977aeef715e9a5cdb0bb
-  data.tar.gz: 39891db9a0400196ddaba587e7014aa00e485c65ce37efc997998f854dd98d8cabfef39806657bec15e71c43a29743dda4081b35c2022a917a5503e4e161cea4
+  metadata.gz: a536a0bf7de4959ca820afa5e9249024181f198d073ca648f040f8f4276f63572ac74a1cdff481b9e050b1983f7af7eac5b2d31c3271dff911ae88aa75092f27
+  data.tar.gz: eddaa254e6254727f950006b30b653402a33acd827b1060baf583840d99ae2d41dec0ab4a15c11754c03db3f2664ea348efefc1b32b95b494c56287a0d3b88d0

data/README.md

@@ -58,7 +58,6 @@ Options:
   --port=VALUE, -p VALUE            # Server port. Overrides SPECWRK_SRV_PORT, default: "5138"
   --bind=VALUE, -b VALUE            # Server bind address. Overrides SPECWRK_SRV_BIND, default: "127.0.0.1"
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]single-seed-per-run        # Only allow one seed per run. Useful for CI where many nodes may seed at the same time, default: false
   --[no-]verbose                    # Run in verbose mode. Default false., default: false
   --help, -h                        # Print this help
 ```
@@ -83,7 +82,6 @@ Options:
   --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY, default: ""
   --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
   --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY: (file/timings), default: "timings"
-  --[no-]single-seed-per-run        # Only allow one seed per run. Useful for CI where many nodes may seed at the same time, default: false
   --[no-]verbose                    # Run in verbose mode. Default false., default: false
   --[no-]single-run                 # Act on shutdown requests from clients. Default: false., default: false
   --help, -h                        # Print this help

data/lib/specwrk/cli.rb

@@ -76,7 +76,6 @@ module Specwrk
         base.unique_option :key, type: :string, aliases: ["-k"], default: ENV.fetch("SPECWRK_SRV_KEY", ""), desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
         base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker output is stored. Overrides SPECWRK_OUT"
         base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWERK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY"
-        base.unique_option :single_seed_per_run, type: :boolean, default: false, desc: "Only allow one seed per run. Useful for CI where many nodes may seed at the same time"
         base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode. Default false."
       end
 
@@ -211,7 +210,8 @@ module Specwrk
           status "Server responding ✓"
           status "Seeding #{examples.length} examples..."
           Client.new.seed(examples)
-          status "Samples seeded ✓"
+          file_count = examples.group_by { |e| e[:file_path] }.keys.size
+          status "🌱 Seeded #{examples.size} examples across #{file_count} files"
         end
 
         if Specwrk.wait_for_pids_exit([seed_pid]).value?(1)

data/lib/specwrk/client.rb

@@ -105,6 +105,23 @@ module Specwrk
       (response.code == "200") ? true : raise(UnhandledResponseError.new("#{response.code}: #{response.body}"))
     end
 
+    def complete_and_fetch_examples(examples)
+      response = post "/complete_and_pop", body: examples.to_json
+
+      case response.code
+      when "200"
+        JSON.parse(response.body, symbolize_names: true)
+      when "204"
+        raise WaitingForSeedError
+      when "404"
+        raise NoMoreExamplesError
+      when "410"
+        raise CompletedAllExamplesError
+      else
+        raise UnhandledResponseError.new("#{response.code}: #{response.body}")
+      end
+    end
+
     def seed(examples)
       response = post "/seed", body: examples.to_json
 

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.9.1"
+  VERSION = "0.10.0"
 end

data/lib/specwrk/web/app.rb

@@ -90,6 +90,8 @@ module Specwrk
           Endpoints::Pop
         when ["POST", "/complete"]
           Endpoints::Complete
+        when ["POST", "/complete_and_pop"]
+          Endpoints::CompleteAndPop
         when ["POST", "/seed"]
           Endpoints::Seed
         when ["GET", "/report"]

data/lib/specwrk/web/endpoints.rb

@@ -137,17 +137,15 @@ module Specwrk
 
       class Seed < Base
         def before_lock
-          examples_with_run_times if persist_seeds?
+          examples_with_run_times
         end
 
         def with_response
-          if persist_seeds?
-            new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
-            pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
-
-            pending.merge!(examples_with_run_times)
-          end
+          pending.clear
+          new_run_time_bucket_maximums = [pending.run_time_bucket_maximum, @seeds_run_time_bucket_maximum.to_f].compact
+          pending.run_time_bucket_maximum = new_run_time_bucket_maximums.sum.to_f / new_run_time_bucket_maximums.length.to_f
 
+          pending.merge!(examples_with_run_times)
           processing.clear
           completed.clear
 
@@ -192,10 +190,6 @@ module Specwrk
           (mean + Math.sqrt(variance)).round(2)
         end
 
-        def persist_seeds?
-          ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"].nil? || pending.empty?
-        end
-
         def sort_by
           if ENV["SPECWRK_SRV_GROUP_BY"] == "file" || run_times.empty?
             :file
@@ -207,6 +201,7 @@ module Specwrk
 
       class Complete < Base
         def with_response
+          warn "[DEPRECATED] This endpoint will be retired in favor of CompleteAndPop. Upgrade your clients."
           completed.merge!(completed_examples)
           processing.delete(*completed_examples.keys)
 
@@ -246,6 +241,51 @@ module Specwrk
         end
       end
 
+      class CompleteAndPop < Base
+        def with_response
+          completed.merge!(completed_examples)
+          run_times.merge! run_time_data
+          processing.delete(*completed_examples.keys)
+
+          @examples = pending.shift_bucket
+
+          processing_data = @examples.map { |example| [example[:id], example] }.to_h
+          processing.merge!(processing_data)
+
+          if @examples.any?
+            [200, {"Content-Type" => "application/json"}, [JSON.generate(@examples)]]
+          elsif pending.empty? && processing.empty? && completed.empty?
+            [204, {"Content-Type" => "text/plain"}, ["Waiting for sample to be seeded."]]
+          elsif completed.any? && processing.empty?
+            [410, {"Content-Type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
+          else
+            not_found
+          end
+        end
+
+        private
+
+        def before_lock
+          completed_examples
+          run_time_data
+        end
+
+        def completed_examples
+          @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
+        end
+
+        # We don't care about exact values here, just approximate run times are fine
+        # So if we overwrite run times from another process it is nbd
+        def after_lock
+          # run_time_data = payload.map { |example| [example[:id], example[:run_time]] }.to_h
+          # run_times.merge! run_time_data
+        end
+
+        def run_time_data
+          @run_time_data ||= payload.map { |example| [example[:id], example[:run_time]] }.to_h
+        end
+      end
+
       class Report < Base
         def with_response
           [200, {"Content-Type" => "application/json"}, [JSON.generate(completed.dump)]]

data/lib/specwrk/worker.rb

@@ -69,15 +69,21 @@ module Specwrk
     end
 
     def execute
-      executor.run client.fetch_examples
+      executor.run next_examples
       complete_examples
     rescue UnhandledResponseError => e
-      # If fetching examples fails we can just try again so warn and return
+      # If fetching examples via next_exampels fails we can just try again so warn and return
+      # Expects complete_examples to rescue this error if raised in that method
       warn e.message
     end
 
+    def next_examples
+      return @next_examples if @next_examples&.length&.positive?
+      client.fetch_examples
+    end
+
     def complete_examples
-      client.complete_examples executor.examples
+      @next_examples = client.complete_and_fetch_examples executor.examples
     rescue UnhandledResponseError => e
       # I do not think we should so lightly abandon the completion of executed examples
       # try to complete until successful or terminated

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.10.0
 platform: ruby
 authors:
 - Daniel Westendorf