specwrk 0.4.9 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5dc867c1250644f28442d434b2932e0798b65cb13398067d95fde60436f97b43
-  data.tar.gz: 824d06bdb7b4af68263bd02e366cd8fe4d728b85cf5df6a810f191d4a7b2c92a
+  metadata.gz: 331e0982599a516c3c862c4a5c33a825c71256f5f95e6d1656c0ad410541df06
+  data.tar.gz: c7d93bf3add6e6de08cad22f8036ef89b01752b0e6dfe42615035d9e301ecff5
 SHA512:
-  metadata.gz: 505174406cb0174bdabe365c5c2a64bebaf98b77ddfe3d5ccad14c922287eb21fdada54c0ab044407f1039e4c01ccdd0487a7120a0c1751d8ce902cdd1efd60c
-  data.tar.gz: f5c49b22baac3d0416b0927db2e2dafc2e4c89a8de88e2fada96daff2e039a4b5a0955dc8489afcf485a2f09ed0164523170c39efeb954a7962d2bddc12d4a85
+  metadata.gz: 9daa901f8a7b7f453079f766d09e47fb621cf85e1cdb6111303e1f8b17cc3091312b2128ca141328f1c8a28216151b991f431cbb4550445ce137349dcb30f3c5
+  data.tar.gz: f0ae841735759d62e9ad6607065ca7bc8747a9d85e8906dea19019af7c63183df26b82cce51bc2a8c453cf4e1bb7d3d0b0c900c917cf689730cdac6028efaae5

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 ## [Unreleased]
 
+## v0.4.11
+
+- Move logic out of CLI methods ([#64](https://github.com/danielwestendorf/specwrk/issues/64)) by @danielwestendorf  
+- Add thruster in front of puma ([#65](https://github.com/danielwestendorf/specwrk/issues/65)) by @danielwestendorf  
+- Revise heartbeats logic ([#66](https://github.com/danielwestendorf/specwrk/issues/66)) by @danielwestendorf  
+
+## v0.4.9
+
+- Remove single-run env var (it was not what I wanted) by @danielwestendorf  
+
+## v0.4.8
+
+- Fix `config.ru` by @danielwestendorf  
+
+## v0.4.7
+
+- Switch to Puma for the Docker image (#59) by @danielwestendorf  
+
+## v0.4.6
+
+- Nil env var that will prevent start command from completing ([#56](https://github.com/danielwestendorf/specwrk/issues/56)) by @danielwestendorf  
+- Better handling of seed failing ([#57](https://github.com/danielwestendorf/specwrk/issues/57)) by @danielwestendorf  
+- Silence health logging ([#58](https://github.com/danielwestendorf/specwrk/issues/58)) by @danielwestendorf  
+- Add missing CCI caching of `report.json` ([#55](https://github.com/danielwestendorf/specwrk/issues/55)) by @danielwestendorf  
+- Add CircleCI examples ([#50](https://github.com/danielwestendorf/specwrk/issues/50)) by @danielwestendorf  
+- Better GHA Examples ([#49](https://github.com/danielwestendorf/specwrk/issues/49)) by @danielwestendorf  
+- Skip key lookup and rely on the result of `Hash#delete` instead by @danielwestendorf  
+
 ## v0.4.5
 
 - Set ENV var when generating seed examples [#47](https://github.com/danielwestendorf/specwrk/issues/47). by @danielwestendorf 

data/README.md

@@ -129,7 +129,7 @@ Description:
   Start one or more worker processes
 
 Options:
-  --id=VALUE                        # The identifier for this worker. Default specwrk-worker(-COUNT_INDEX), default: "specwrk-worker"
+  --id=VALUE                        # The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used
   --count=VALUE, -c VALUE           # The number of worker processes you want to start, default: 1
   --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT, default: ".specwrk/"
   --seed-waits=VALUE, -w VALUE      # Number of times the worker will wait for examples to be seeded to the server. 1sec between attempts. Overrides SPECWRK_SEED_WAITS, default: "10"

data/config.ru

@@ -2,4 +2,5 @@ require "specwrk"
 require "specwrk/web"
 require "specwrk/web/app"
 
+Specwrk::Web::App.setup!
 run Specwrk::Web::App.rackup

data/docker/Dockerfile.server

@@ -19,7 +19,7 @@ COPY $GEMFILE ./
 RUN gem install ./$GEMFILE --no-document
 RUN rm ./$GEMFILE
 
-RUN gem install puma
+RUN gem install puma thruster
 COPY config.ru ./
 
 COPY docker/entrypoint.server.sh /usr/local/bin/entrypoint

data/docker/entrypoint.server.sh

@@ -1,3 +1,7 @@
 #!/bin/sh
 
-exec puma --port ${PORT:-$SPECWRK_SRV_PORT} --bind tcp://0.0.0.0 --threads 1 --workers 0 --silent
+export THRUSTER_HTTP_PORT=${PORT:-5138}
+export THRUSTER_TARGET_PORT=3000
+export THRUSTER_HTTP_IDLE_TIMEOUT=${IDLE_TIMEOUT:-305}
+
+exec thrust puma --workers 0 --bind tcp://127.0.0.1:3000 --threads ${PUMA_THREADS:-1}

data/lib/specwrk/cli.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "pathname"
-require "fileutils"
+require "securerandom"
 
 require "dry/cli"
 
@@ -34,18 +34,18 @@ module Specwrk
       extend Hookable
 
       on_included do |base|
-        base.unique_option :id, type: :string, default: "specwrk-worker", desc: "The identifier for this worker. Default specwrk-worker(-COUNT_INDEX)"
+        base.unique_option :id, type: :string, desc: "The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used"
         base.unique_option :count, type: :integer, default: 1, aliases: ["-c"], desc: "The number of worker processes you want to start"
         base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker output is stored. Overrides SPECWRK_OUT"
         base.unique_option :seed_waits, type: :integer, default: ENV.fetch("SPECWRK_SEED_WAITS", "10"), aliases: ["-w"], desc: "Number of times the worker will wait for examples to be seeded to the server. 1sec between attempts. Overrides SPECWRK_SEED_WAITS"
       end
 
-      on_setup do |id:, count:, output:, seed_waits:, **|
-        ENV["SPECWRK_ID"] = id
+      on_setup do |count:, output:, seed_waits:, id: "specwrk-worker-#{SecureRandom.uuid[0, 8]}", **|
+        ENV["SPECWRK_ID"] ||= id # Unique default. Don't override the ENV value here
+
         ENV["SPECWRK_COUNT"] = count.to_s
         ENV["SPECWRK_SEED_WAITS"] = seed_waits.to_s
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
-        FileUtils.mkdir_p(ENV["SPECWRK_OUT"])
       end
 
       def start_workers
@@ -80,16 +80,13 @@ module Specwrk
         base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode. Default false."
       end
 
-      on_setup do |port:, bind:, output:, key:, single_seed_per_run:, group_by:, verbose:, **|
+      on_setup do |port:, bind:, output:, key:, group_by:, verbose:, **|
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
-        FileUtils.mkdir_p(ENV["SPECWRK_OUT"])
+        ENV["SPECWRK_SRV_VERBOSE"] = "1" if verbose
 
-        ENV["SPECWRK_SRV_LOG"] ||= Pathname.new(File.join(ENV["SPECWRK_OUT"], "server.log")).to_s if output && !verbose
-        ENV["SPECWRK_SRV_OUTPUT"] ||= Pathname.new(File.join(ENV["SPECWRK_OUT"], "report.json")).expand_path(Dir.pwd).to_s if output
         ENV["SPECWRK_SRV_PORT"] = port
         ENV["SPECWRK_SRV_BIND"] = bind
         ENV["SPECWRK_SRV_KEY"] = key
-        ENV["SPECWRK_SRV_SINGLE_SEED_PER_RUN"] = "1" if single_seed_per_run
         ENV["SPECWRK_SRV_GROUP_BY"] = group_by
       end
     end
@@ -120,6 +117,8 @@ module Specwrk
 
         Client.wait_for_server!
         Client.new.seed(examples)
+        file_count = examples.group_by { |e| e[:file_path] }.keys.size
+        puts "🌱 Seeded #{examples.size} examples across #{file_count} files"
       rescue Errno::ECONNREFUSED
         puts "Server at #{ENV.fetch("SPECWRK_SRV_URI", "http://localhost:5138")} is refusing connections, exiting...#{ENV["SPECWRK_FLUSH_DELIMINATOR"]}"
         exit 1

data/lib/specwrk/client.rb

@@ -150,7 +150,9 @@ module Specwrk
 
     def default_headers
       @default_headers ||= {}.tap do |h|
+        h["User-Agent"] = "Specwrk/#{VERSION}"
         h["Authorization"] = "Bearer #{ENV["SPECWRK_SRV_KEY"]}" if ENV["SPECWRK_SRV_KEY"]
+        h["X-Specwrk-Id"] = ENV.fetch("SPECWRK_ID", "specwrk-client")
         h["X-Specwrk-Run"] = ENV["SPECWRK_RUN"] if ENV["SPECWRK_RUN"]
         h["Content-Type"] = "application/json"
       end

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.4.9"
+  VERSION = "0.5.0"
 end

data/lib/specwrk/web/app.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require "pathname"
+require "fileutils"
+
 require "webrick"
 require "rack"
 
@@ -10,6 +13,7 @@ rescue LoadError
   require "rack/handler/webrick"
 end
 
+require "specwrk/web"
 require "specwrk/web/logger"
 require "specwrk/web/auth"
 require "specwrk/web/endpoints"
@@ -17,13 +21,13 @@ require "specwrk/web/endpoints"
 module Specwrk
   class Web
     class App
+      REAP_INTERVAL = 330 # HTTP connection timeout + some buffer
+
       class << self
         def run!
           Process.setproctitle "specwrk-server"
 
-          if ENV["SPECWRK_SRV_LOG"]
-            $stdout.reopen(ENV["SPECWRK_SRV_LOG"], "w")
-          end
+          setup!
 
           server_opts = {
             Port: ENV.fetch("SPECWRK_SRV_PORT", "5138").to_i,
@@ -46,16 +50,36 @@ module Specwrk
           end
         end
 
+        def setup!
+          if ENV["SPECWRK_OUT"]
+
+            FileUtils.mkdir_p(ENV["SPECWRK_OUT"])
+            ENV["SPECWRK_SRV_LOG"] ||= Pathname.new(File.join(ENV["SPECWRK_OUT"], "server.log")).to_s unless ENV["SPECWRK_SRV_VERBOSE"]
+            ENV["SPECWRK_SRV_OUTPUT"] ||= Pathname.new(File.join(ENV["SPECWRK_OUT"], "report.json")).expand_path(Dir.pwd).to_s
+          end
+
+          if ENV["SPECWRK_SRV_LOG"]
+            $stdout.reopen(ENV["SPECWRK_SRV_LOG"], "w")
+          end
+        end
+
         def rackup
           Rack::Builder.new do
-            use Rack::Runtime
-            use Specwrk::Web::Logger, $stdout, %w[/health]
+            if ENV["SPECWRK_SRV_VERBOSE"]
+              use Rack::Runtime
+              use Specwrk::Web::Logger, $stdout, %w[/health]
+            end
+
             use Specwrk::Web::Auth, %w[/health] # global auth check
             run Specwrk::Web::App.new           # your router
           end
         end
       end
 
+      def initialize
+        @reaper_thread = Thread.new { reaper } unless ENV["SPECWRK_SRV_SINGLE_RUN"]
+      end
+
       def call(env)
         env[:request] ||= Rack::Request.new(env)
 
@@ -84,6 +108,26 @@ module Specwrk
           Endpoints::NotFound
         end
       end
+
+      def reaper
+        until Specwrk.force_quit
+          sleep REAP_INTERVAL
+
+          reap
+        end
+      end
+
+      def reap
+        Web::WORKERS.each do |run, workers|
+          most_recent_last_seen_at = workers.map { |id, worker| worker[:last_seen_at] }.max
+          next unless most_recent_last_seen_at
+
+          # Don't consider runs which aren't at least REAP_INTERVAL sec stale
+          if most_recent_last_seen_at < Time.now - REAP_INTERVAL
+            Web.clear_run_queues(run)
+          end
+        end
+      end
     end
   end
 end

data/lib/specwrk/web/endpoints.rb

@@ -8,6 +8,9 @@ module Specwrk
       class Base
         def initialize(request)
           @request = request
+
+          worker[:first_seen_at] ||= Time.now
+          worker[:last_seen_at] = Time.now
         end
 
         def response
@@ -45,6 +48,14 @@ module Specwrk
         def completed_queue
           Web::COMPLETED_QUEUES[request.get_header("HTTP_X_SPECWRK_RUN")]
         end
+
+        def workers
+          Web::WORKERS[request.get_header("HTTP_X_SPECWRK_RUN")]
+        end
+
+        def worker
+          workers[request.get_header("HTTP_X_SPECWRK_ID")]
+        end
       end
 
       # Base default response is 404
@@ -135,25 +146,7 @@ module Specwrk
 
       class Shutdown < Base
         def response
-          if ENV["SPECWRK_SRV_SINGLE_RUN"]
-            interupt!
-          elsif processing_queue.length.positive?
-            # Push any processing jobs back into the pending queue
-            processing_queue.synchronize do |processing_queue_hash|
-              pending_queue.synchronize do |pending_queue_hash|
-                processing_queue_hash.each do |id, example|
-                  pending_queue_hash[id] = example
-                  processing_queue_hash.delete(id)
-                end
-              end
-            end
-
-          elsif processing_queue.length.zero? && pending_queue.length.zero?
-            # All done, we can clear the completed queue
-            completed_queue.clear
-          end
-
-          # TODO: clear any zombie queues
+          interupt! if ENV["SPECWRK_SRV_SINGLE_RUN"]
 
           [200, {"Content-Type" => "text/plain"}, ["✌️"]]
         end

data/lib/specwrk/web.rb

@@ -7,9 +7,16 @@ module Specwrk
     PENDING_QUEUES = Queue.new { |h, key| h[key] = PendingQueue.new.tap { |q| q.previous_run_times_file = ENV["SPECWRK_SRV_OUTPUT"] } }
     PROCESSING_QUEUES = Queue.new { |h, key| h[key] = Queue.new }
     COMPLETED_QUEUES = Queue.new { |h, key| h[key] = CompletedQueue.new }
+    WORKERS = Hash.new { |h, key| h[key] = Hash.new { |h, key| h[key] = {} } }
 
     def self.clear_queues
-      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES].each(&:clear)
+      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each(&:clear)
+    end
+
+    def self.clear_run_queues(run)
+      [PENDING_QUEUES, PROCESSING_QUEUES, COMPLETED_QUEUES, WORKERS].each do |queue|
+        queue.delete(run)
+      end
     end
   end
 end

data/lib/specwrk/worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "stringio"
+require "fileutils"
 
 require "specwrk/client"
 require "specwrk/worker/executor"
@@ -13,6 +14,7 @@ module Specwrk
 
     def initialize
       Process.setproctitle ENV.fetch("SPECWRK_ID", "specwrk-worker")
+      FileUtils.mkdir_p(ENV["SPECWRK_OUT"]) if ENV["SPECWRK_OUT"]
 
       @running = true
       @client = Client.new
@@ -85,13 +87,13 @@ module Specwrk
 
     def thump
       while running && !Specwrk.force_quit
+        sleep 10
+
         begin
-          client.heartbeat if client.last_request_at.nil? || client.last_request_at < Time.now - 10
+          client.heartbeat if client.last_request_at.nil? || client.last_request_at < Time.now - 30
         rescue
           warn "Heartbeat failed!"
         end
-
-        sleep 1
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.5.0
 platform: ruby
 authors:
 - Daniel Westendorf