specwrk 0.11.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f5fa0464c4d73ad78593cd2ce5626dac594bb74721d18303f1fee4d33e28615
-  data.tar.gz: 262e478389d7dd07375fc5a864012c992709199838a2f32f7e72e09bbb412759
+  metadata.gz: c7b5abc9ec00673b175314cce03016affa2e47b7ca9f445235ea93fdadb7c79a
+  data.tar.gz: 7f30c131e7bbce5c787c527af0e1b58acb16345cc85c5cf22671be1281e8106e
 SHA512:
-  metadata.gz: 21fda87f804947653887bcd897c45c97ebdc9227a63bc19019b137e01955dc96701f202f1ebe8a472ed3b871e799a52f54ab9b2872f4f76a2ebefc4e411801df
-  data.tar.gz: 7a6a62993e9a1262a50c291021fdb1ec6662ed36453bc0ec3855aa7a01735647943500e354a6705c1b6e1f190e3bd0f297bd2317cc639d9fa576a115c4061e30
+  metadata.gz: 75d4427606c9491a4344d95713f2062df258e7aea660e520ee126dc2233b8b2bd2b24ca9f56c15d696b0b35b9c0a6211ac05c2b4e63af950df5723e0c625816e
+  data.tar.gz: '0885f9962b4cc30f84d72d8e0a99dbb3541588f6aefc292657d95f52576c80251c2cfc4d04af2f35f74df74ca09ebb5385493d55f67ff905cd7cb1522c99f1b4'

data/lib/specwrk/cli_reporter.rb

@@ -16,8 +16,8 @@ module Specwrk
         return 1
       end
 
-      puts "\nFinished in #{total_duration} " \
-                          "(total execution time of #{total_run_time})\n"
+      puts "\nFinished in #{Specwrk.human_readable_duration total_duration} " \
+                          "(total execution time of #{Specwrk.human_readable_duration total_run_time})\n"
 
       client.shutdown
 

data/lib/specwrk/client.rb

@@ -44,12 +44,13 @@ module Specwrk
       raise Errno::ECONNREFUSED unless connected
     end
 
-    attr_reader :last_request_at, :retry_count
+    attr_reader :last_request_at, :retry_count, :worker_status
 
     def initialize
       @mutex = Mutex.new
       @http = self.class.build_http
       @http.start
+      @worker_status = 1
     end
 
     def close
@@ -99,12 +100,6 @@ module Specwrk
       end
     end
 
-    def complete_examples(examples)
-      response = post "/complete", body: examples.to_json
-
-      (response.code == "200") ? true : raise(UnhandledResponseError.new("#{response.code}: #{response.body}"))
-    end
-
     def complete_and_fetch_examples(examples)
       response = post "/complete_and_pop", body: examples.to_json
 
@@ -171,7 +166,11 @@ module Specwrk
     def make_request(request)
       @mutex.synchronize do
         @last_request_at = Time.now
-        @http.request(request).tap { @retry_count = 0 }
+        @http.request(request).tap do |response|
+          @retry_count = 0
+
+          @worker_status = response["x-specwrk-status"].to_i if response["x-specwrk-status"]
+        end
       end
     end
 

data/lib/specwrk/store/file_adapter.rb

@@ -60,7 +60,6 @@ module Specwrk
         else
           filename = filename_for_key(key_string)
           write(filename, JSON.generate(value))
-          known_key_pairs[key_string] = filename
         end
       end
 
@@ -71,16 +70,12 @@ module Specwrk
       def clear
         FileUtils.rm_rf(path)
         FileUtils.mkdir_p(path)
-
-        @known_key_pairs = nil
       end
 
       def delete(*keys)
-        filenames = keys.map { |key| known_key_pairs[key] }.compact
+        filenames = keys.map { |key| filename_for_key key }.compact
 
         FileUtils.rm_f(filenames)
-
-        keys.each { |key| known_key_pairs.delete(key) }
       end
 
       def merge!(h2)
@@ -88,8 +83,6 @@ module Specwrk
       end
 
       def multi_read(*read_keys)
-        known_key_pairs # precache before each thread tries to look them up
-
         result_queue = Queue.new
 
         read_keys.each do |key|
@@ -112,8 +105,6 @@ module Specwrk
       end
 
       def multi_write(hash)
-        known_key_pairs # precache before each thread tries to look them up
-
         result_queue = Queue.new
 
         hash_with_filenames = hash.map { |key, value| [key.to_s, [filename_for_key(key.to_s), value]] }.to_h
@@ -126,7 +117,6 @@ module Specwrk
         end
 
         Thread.pass until result_queue.length == hash.length
-        hash_with_filenames.each { |key, (filename, _value)| known_key_pairs[key] = filename }
       end
 
       def empty?
@@ -136,7 +126,7 @@ module Specwrk
       private
 
       def write(filename, content)
-        tmp_filename = [filename, "tmp"].join(".")
+        tmp_filename = [filename, SecureRandom.uuid, "tmp"].join(".")
 
         File.binwrite(tmp_filename, content)
 
@@ -145,29 +135,19 @@ module Specwrk
       end
 
       def read(key)
-        File.read(known_key_pairs[key]) if known_key_pairs.key? key
+        filename = filename_for_key key
+        File.read(filename)
+      rescue Errno::ENOENT
+        nil
       end
 
       def filename_for_key(key)
         File.join(
           path,
-          [
-            counter_prefix(key),
-            encode_key(key)
-          ].join("_")
+          encode_key(key)
         ) + EXT
       end
 
-      def counter_prefix(key)
-        count = keys.index(key) || counter.tap { @counter += 1 }
-
-        "%012d" % count
-      end
-
-      def counter
-        @counter ||= keys.length
-      end
-
       def path
         @path ||= File.join(uri.path, scope).tap do |full_path|
           FileUtils.mkdir_p(full_path)
@@ -179,14 +159,14 @@ module Specwrk
       end
 
       def decode_key(key)
-        encoded_key_part = File.basename(key).delete_suffix(EXT).split(/\A\d+_/).last
+        encoded_key_part = File.basename(key).delete_suffix(EXT)
         padding_count = (4 - encoded_key_part.length % 4) % 4
 
         Base64.urlsafe_decode64(encoded_key_part + ("=" * padding_count))
       end
 
       def known_key_pairs
-        @known_key_pairs ||= Dir.entries(path).sort.map do |filename|
+        Dir.entries(path).sort.map do |filename|
           next if filename.start_with? "."
           next unless filename.end_with? EXT
 

data/lib/specwrk/store.rb

@@ -72,7 +72,7 @@ module Specwrk
     end
 
     def to_h
-      adapter.multi_read(*keys).transform_keys!(&:to_sym)
+      multi_read(*keys).transform_keys!(&:to_sym)
     end
 
     def inspect
@@ -98,6 +98,7 @@ module Specwrk
 
   class PendingStore < Store
     RUN_TIME_BUCKET_MAXIMUM_KEY = :____run_time_bucket_maximum
+    ORDER_KEY = :____order
 
     def run_time_bucket_maximum=(val)
       @run_time_bucket_maximum = self[RUN_TIME_BUCKET_MAXIMUM_KEY] = val
@@ -107,6 +108,41 @@ module Specwrk
       @run_time_bucket_maximum ||= self[RUN_TIME_BUCKET_MAXIMUM_KEY]
     end
 
+    def order=(val)
+      @order = nil
+
+      self[ORDER_KEY] = if val.nil? || val.length.zero?
+        nil
+      else
+        val
+      end
+    end
+
+    def order
+      @order ||= self[ORDER_KEY] || []
+    end
+
+    def keys
+      return super if order.length.zero?
+
+      order
+    end
+
+    def merge!(hash)
+      super
+      self.order = hash.keys
+    end
+
+    def clear
+      @order = nil
+      super
+    end
+
+    def reload
+      @order = nil
+      super
+    end
+
     def shift_bucket
       return bucket_by_file unless run_time_bucket_maximum&.positive?
 
@@ -146,6 +182,7 @@ module Specwrk
       end
 
       delete(*consumed_keys)
+      self.order = order - consumed_keys
       bucket
     end
 
@@ -171,6 +208,7 @@ module Specwrk
       end
 
       delete(*consumed_keys)
+      self.order = order - consumed_keys
       bucket
     end
   end

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.11.0"
+  VERSION = "0.13.0"
 end

data/lib/specwrk/web/app.rb

@@ -88,8 +88,6 @@ module Specwrk
           Endpoints::Heartbeat
         when ["POST", "/pop"]
           Endpoints::Pop
-        when ["POST", "/complete"]
-          Endpoints::Complete
         when ["POST", "/complete_and_pop"]
           Endpoints::CompleteAndPop
         when ["POST", "/seed"]

data/lib/specwrk/web/auth.rb

@@ -11,7 +11,7 @@ module Specwrk
       end
 
       def call(env)
-        env[:request] ||= Rack::Request.new(env)
+        @request = env[:request] ||= Rack::Request.new(env)
 
         return @app.call(env) if [nil, ""].include? ENV["SPECWRK_SRV_KEY"]
         return @app.call(env) if @excluded_paths.include? env[:request].path_info
@@ -28,7 +28,11 @@ module Specwrk
       private
 
       def unauthorized
-        [401, {"content-type" => "application/json"}, ["Unauthorized"]]
+        if @request.head?
+          [401, {}, []]
+        else
+          [401, {"content-type" => "application/json"}, ["Unauthorized"]]
+        end
       end
     end
   end

data/lib/specwrk/web/endpoints.rb

@@ -33,6 +33,8 @@ module Specwrk
 
           after_lock
 
+          final_response[1]["x-specwrk-status"] = worker_status.to_s
+
           final_response
         end
 
@@ -102,6 +104,12 @@ module Specwrk
           @worker ||= Store.new(ENV.fetch("SPECWRK_SRV_STORE_URI", "memory:///"), File.join(run_id, "workers", request.get_header("HTTP_X_SPECWRK_ID").to_s))
         end
 
+        def worker_status
+          return 0 if worker[:failed].nil? && completed.any? # worker starts after run has completed
+
+          worker[:failed] || 1
+        end
+
         def run_id
           request.get_header("HTTP_X_SPECWRK_RUN")
         end
@@ -190,29 +198,6 @@ module Specwrk
         end
       end
 
-      class Complete < Base
-        def with_response
-          warn "[DEPRECATED] This endpoint will be retired in favor of CompleteAndPop. Upgrade your clients."
-          completed.merge!(completed_examples)
-          processing.delete(*completed_examples.keys)
-
-          ok
-        end
-
-        private
-
-        def completed_examples
-          @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
-        end
-
-        # We don't care about exact values here, just approximate run times are fine
-        # So if we overwrite run times from another process it is nbd
-        def after_lock
-          run_time_data = payload.map { |example| [example[:id], example[:run_time]] }.to_h
-          run_times.merge! run_time_data
-        end
-      end
-
       class Popable < Base
         private
 
@@ -261,6 +246,8 @@ module Specwrk
       end
 
       class CompleteAndPop < Popable
+        EXAMPLE_STATUSES = %w[passed failed pending]
+
         def with_response
           completed.merge!(completed_examples)
           processing.delete(*completed_examples.keys)
@@ -270,19 +257,26 @@ module Specwrk
 
         private
 
-        def before_lock
-          completed_examples
-          run_time_data
-        end
-
         def completed_examples
           @completed_data ||= payload.map { |example| [example[:id], example] if processing[example[:id]] }.compact.to_h
         end
 
-        # We don't care about exact values here, just approximate run times are fine
-        # So if we overwrite run times from another process it is nbd
+        def completed_examples_status_counts
+          @completed_examples_status_counts ||= completed_examples.values.map { |example| example[:status] }.tally
+        end
+
         def after_lock
+          # We don't care about exact values here, just approximate run times are fine
+          # So if we overwrite run times from another process it is nbd
           run_times.merge! run_time_data
+
+          # workers are single proces, single-threaded, so safe to do this work without the lock
+          existing_status_counts = worker.multi_read(*EXAMPLE_STATUSES)
+          new_status_counts = EXAMPLE_STATUSES.map do |status|
+            [status, existing_status_counts.fetch(status, 0) + completed_examples_status_counts.fetch(status, 0)]
+          end.to_h
+
+          worker.merge!(new_status_counts)
         end
 
         def run_time_data

data/lib/specwrk/worker/completion_formatter.rb

@@ -5,19 +5,14 @@ module Specwrk
     class CompletionFormatter
       RSpec::Core::Formatters.register self, :stop
 
-      attr_reader :examples, :failure
+      attr_reader :examples
 
       def initialize
         @examples = []
-        @failure = false
       end
 
       def stop(group_notification)
         group_notification.notifications.map do |notification|
-          unless failure
-            @failure = notification.example.execution_result.status == :failed
-          end
-
           examples << {
             id: notification.example.id,
             full_description: notification.example.full_description,

data/lib/specwrk/worker/executor.rb

@@ -11,12 +11,6 @@ require "specwrk/worker/null_formatter"
 module Specwrk
   class Worker
     class Executor
-      attr_reader :example_processed
-
-      def failure
-        completion_formatter.failure
-      end
-
       def examples
         completion_formatter.examples
       end
@@ -29,7 +23,6 @@ module Specwrk
         reset!
 
         example_ids = examples.map { |example| example[:id] }
-        @example_processed ||= true unless example_ids.size.zero? # only ever toggle from nil => true
 
         options = RSpec::Core::ConfigurationOptions.new rspec_options + example_ids
         RSpec::Core::Runner.new(options).run($stderr, $stdout)
@@ -40,6 +33,7 @@ module Specwrk
         completion_formatter.examples.clear
 
         RSpec.clear_examples
+        RSpec.configuration.backtrace_formatter.filter_gem "specwrk"
 
         # see https://github.com/rspec/rspec-core/pull/2723
         if Gem::Version.new(RSpec::Core::Version::STRING) <= Gem::Version.new("3.9.1")

data/lib/specwrk/worker.rb

@@ -110,11 +110,10 @@ module Specwrk
     attr_reader :running, :client, :executor
 
     def status
-      return 1 if !executor.example_processed && !@all_examples_completed
-      return 1 if executor.failure
+      return 0 if @all_examples_completed && client.worker_status.zero?
       return 1 if Specwrk.force_quit
 
-      0
+      client.worker_status
     end
 
     def warn(msg)

data/lib/specwrk.rb

@@ -38,5 +38,21 @@ module Specwrk
 
       exited_pids
     end
+
+    def human_readable_duration(total_seconds, precision: 2)
+      secs = total_seconds.to_f
+      hours = (secs / 3600).to_i
+      mins = ((secs % 3600) / 60).to_i
+      seconds = secs % 60
+
+      parts = []
+      parts << "#{hours}h" if hours.positive?
+      parts << "#{mins}m" if mins.positive?
+      if seconds.positive?
+        sec_str = format("%0.#{precision}f", seconds).sub(/\.?0+$/, "")
+        parts << "#{sec_str}s"
+      end
+      parts.join(" ")
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Daniel Westendorf