specwrk 0.4.11 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e85bdff31c7f1312b12a9a823200a5da5b36653433496556cc1bce274a3dc90a
-  data.tar.gz: fa73aebb69753d7b528d333f2fca589e0e5525bd12b71e4c5b7815df0932b9a1
+  metadata.gz: 331e0982599a516c3c862c4a5c33a825c71256f5f95e6d1656c0ad410541df06
+  data.tar.gz: c7d93bf3add6e6de08cad22f8036ef89b01752b0e6dfe42615035d9e301ecff5
 SHA512:
-  metadata.gz: 9467744c0cd676bdeb312b3f1deb4190113601cb01bb9ce939acdfa1c6170897904f3125b43c90dde536459ed56b2623be132bfeed84725c1f6b098ea65ae6a9
-  data.tar.gz: f8cd7800b9f117567c1aa53404f2ca7f9a99a907c4b46c1c05da34d7534a2743c68960c50ffd9555de82430f582cb6a042a70579e4a8a44f53dea136f72cd8ae
+  metadata.gz: 9daa901f8a7b7f453079f766d09e47fb621cf85e1cdb6111303e1f8b17cc3091312b2128ca141328f1c8a28216151b991f431cbb4550445ce137349dcb30f3c5
+  data.tar.gz: f0ae841735759d62e9ad6607065ca7bc8747a9d85e8906dea19019af7c63183df26b82cce51bc2a8c453cf4e1bb7d3d0b0c900c917cf689730cdac6028efaae5

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 ## [Unreleased]
 
+## v0.4.11
+
+- Move logic out of CLI methods ([#64](https://github.com/danielwestendorf/specwrk/issues/64)) by @danielwestendorf  
+- Add thruster in front of puma ([#65](https://github.com/danielwestendorf/specwrk/issues/65)) by @danielwestendorf  
+- Revise heartbeats logic ([#66](https://github.com/danielwestendorf/specwrk/issues/66)) by @danielwestendorf  
+
+## v0.4.9
+
+- Remove single-run env var (it was not what I wanted) by @danielwestendorf  
+
+## v0.4.8
+
+- Fix `config.ru` by @danielwestendorf  
+
+## v0.4.7
+
+- Switch to Puma for the Docker image (#59) by @danielwestendorf  
+
+## v0.4.6
+
+- Nil env var that will prevent start command from completing ([#56](https://github.com/danielwestendorf/specwrk/issues/56)) by @danielwestendorf  
+- Better handling of seed failing ([#57](https://github.com/danielwestendorf/specwrk/issues/57)) by @danielwestendorf  
+- Silence health logging ([#58](https://github.com/danielwestendorf/specwrk/issues/58)) by @danielwestendorf  
+- Add missing CCI caching of `report.json` ([#55](https://github.com/danielwestendorf/specwrk/issues/55)) by @danielwestendorf  
+- Add CircleCI examples ([#50](https://github.com/danielwestendorf/specwrk/issues/50)) by @danielwestendorf  
+- Better GHA Examples ([#49](https://github.com/danielwestendorf/specwrk/issues/49)) by @danielwestendorf  
+- Skip key lookup and rely on the result of `Hash#delete` instead by @danielwestendorf  
+
 ## v0.4.5
 
 - Set ENV var when generating seed examples [#47](https://github.com/danielwestendorf/specwrk/issues/47). by @danielwestendorf 

data/README.md

@@ -129,7 +129,7 @@ Description:
   Start one or more worker processes
 
 Options:
-  --id=VALUE                        # The identifier for this worker. Default specwrk-worker(-COUNT_INDEX), default: "specwrk-worker"
+  --id=VALUE                        # The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used
   --count=VALUE, -c VALUE           # The number of worker processes you want to start, default: 1
   --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
   --seed-waits=VALUE, -w VALUE      # Number of times the worker will wait for examples to be seeded to the server. 1sec between attempts. Overrides SPECWRK_SEED_WAITS, default: "10"

data/lib/specwrk/cli.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "pathname"
+require "securerandom"
 
 require "dry/cli"
 
@@ -33,14 +34,15 @@ module Specwrk
       extend Hookable
 
       on_included do |base|
-        base.unique_option :id, type: :string, default: "specwrk-worker", desc: "The identifier for this worker. Default specwrk-worker(-COUNT_INDEX)"
+        base.unique_option :id, type: :string, desc: "The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used"
         base.unique_option :count, type: :integer, default: 1, aliases: ["-c"], desc: "The number of worker processes you want to start"
         base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker output is stored. Overrides SPECWRK_OUT"
         base.unique_option :seed_waits, type: :integer, default: ENV.fetch("SPECWRK_SEED_WAITS", "10"), aliases: ["-w"], desc: "Number of times the worker will wait for examples to be seeded to the server. 1sec between attempts. Overrides SPECWRK_SEED_WAITS"
       end
 
-      on_setup do |id:, count:, output:, seed_waits:, **|
-        ENV["SPECWRK_ID"] = id
+      on_setup do |count:, output:, seed_waits:, id: "specwrk-worker-#{SecureRandom.uuid[0, 8]}", **|
+        ENV["SPECWRK_ID"] ||= id # Unique default. Don't override the ENV value here
+
         ENV["SPECWRK_COUNT"] = count.to_s
         ENV["SPECWRK_SEED_WAITS"] = seed_waits.to_s
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
@@ -115,6 +117,8 @@ module Specwrk
 
         Client.wait_for_server!
         Client.new.seed(examples)
+        file_count = examples.group_by { |e| e[:file_path] }.keys.size
+        puts "🌱 Seeded #{examples.size} examples across #{file_count} files"
       rescue Errno::ECONNREFUSED
         puts "Server at #{ENV.fetch("SPECWRK_SRV_URI", "http://localhost:5138")} is refusing connections, exiting...#{ENV["SPECWRK_FLUSH_DELIMINATOR"]}"
         exit 1

data/lib/specwrk/client.rb

@@ -150,7 +150,9 @@ module Specwrk
 
     def default_headers
       @default_headers ||= {}.tap do |h|
+        h["User-Agent"] = "Specwrk/#{VERSION}"
         h["Authorization"] = "Bearer #{ENV["SPECWRK_SRV_KEY"]}" if ENV["SPECWRK_SRV_KEY"]
+        h["X-Specwrk-Id"] = ENV.fetch("SPECWRK_ID", "specwrk-client")
         h["X-Specwrk-Run"] = ENV["SPECWRK_RUN"] if ENV["SPECWRK_RUN"]
         h["Content-Type"] = "application/json"
       end

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.4.11"
+  VERSION = "0.5.0"
 end

data/lib/specwrk/web/app.rb

@@ -13,6 +13,7 @@ rescue LoadError
   require "rack/handler/webrick"
 end
 
+require "specwrk/web"
 require "specwrk/web/logger"
 require "specwrk/web/auth"
 require "specwrk/web/endpoints"
@@ -20,6 +21,8 @@ require "specwrk/web/endpoints"
 module Specwrk
   class Web
     class App
+      REAP_INTERVAL = 330 # HTTP connection timeout + some buffer
+
       class << self
         def run!
           Process.setproctitle "specwrk-server"
@@ -73,6 +76,10 @@ module Specwrk
         end
       end
 
+      def initialize
+        @reaper_thread = Thread.new { reaper } unless ENV["SPECWRK_SRV_SINGLE_RUN"]
+      end
+
       def call(env)
         env[:request] ||= Rack::Request.new(env)
 
@@ -101,6 +108,26 @@ module Specwrk
           Endpoints::NotFound
         end
       end
+
+      def reaper
+        until Specwrk.force_quit
+          sleep REAP_INTERVAL
+
+          reap
+        end
+      end
+
+      def reap
+        Web::WORKERS.each do |run, workers|
+          most_recent_last_seen_at = workers.map { |id, worker| worker[:last_seen_at] }.max
+          next unless most_recent_last_seen_at
+
+          # Don't consider runs which aren't at least REAP_INTERVAL sec stale
+          if most_recent_last_seen_at < Time.now - REAP_INTERVAL
+            Web.clear_run_queues(run)
+          end
+        end
+      end
     end
   end
 end

data/lib/specwrk/web/endpoints.rb

@@ -8,6 +8,9 @@ module Specwrk
       class Base
         def initialize(request)
           @request = request
+
+          worker[:first_seen_at] ||= Time.now
+          worker[:last_seen_at] = Time.now
         end
 
         def response
@@ -45,6 +48,14 @@ module Specwrk
         def completed_queue
           Web::COMPLETED_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
         end
+
+        def workers
+          Web::WORKERS[request.get_header("HTTP_X_SPECWRK_RUN")]
+        end
+
+        def worker
+          workers[request.get_header("HTTP_X_SPECWRK_ID")]
+        end
       end
 
       # Base default response is 404
@@ -135,25 +146,7 @@ module Specwrk
 
       class Shutdown < Base
         def response
-          if ENV["SPECWRK_SRV_SINGLE_RUN"]
-            interupt!
-          elsif processing_queue.length.positive?
-            # Push any processing jobs back into the pending queue
-            processing_queue.synchronize do |processing_queue_hash|
-              pending_queue.synchronize do |pending_queue_hash|
-                processing_queue_hash.each do |id, example|
-                  pending_queue_hash[id] = example
-                  processing_queue_hash.delete(id)
-                end
-              end
-            end
-
-          elsif processing_queue.length.zero? && pending_queue.length.zero?
-            # All done, we can clear the completed queue
-            completed_queue.clear
-          end
-
-          # TODO: clear any zombie queues
+          interupt! if ENV["SPECWRK_SRV_SINGLE_RUN"]
 
           [200, {"Content-Type" => "text/plain"}, ["✌️"]]
         end

data/lib/specwrk/web.rb

@@ -7,9 +7,16 @@ module Specwrk
     PENDING_QUEUES = Queue.new { |h, key| h[key] = PendingQueue.new.tap { |q| q.previous_run_times_file = ENV["SPECWRK_SRV_OUTPUT"] } }
     PROCESSING_QUEUES = Queue.new { |h, key| h[key] = Queue.new }
     COMPLETED_QUEUES = Queue.new { |h, key| h[key] = CompletedQueue.new }
+    WORKERS = Hash.new { |h, key| h[key] = Hash.new { |h, key| h[key] = {} } }
 
     def self.clear_queues
-      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES].each(&:clear)
+      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each(&:clear)
+    end
+
+    def self.clear_run_queues(run)
+      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each do |queue|
+        queue.delete(run)
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.4.11
+  version: 0.5.0
 platform: ruby
 authors:
 - Daniel Westendorf