specwrk 0.17.1 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93a17f70f0e768222876b22465e7b62d60deb9dbe9761eed9611f31638a7f7cb
-  data.tar.gz: 9137ab50e74ea4e3e46d74496718ae2830af4bb3c1a51e4b0ac2ba53d1691617
+  metadata.gz: 690eaebf0ec2f7621284bc0ce56d6613001bdffc6297651b9d4f6da7a439971b
+  data.tar.gz: 8d3104699becc543cbe5aaa6f2b6537d931d34d0e931d720b254262d5b2889a9
 SHA512:
-  metadata.gz: cbcbb39d8d270fee9fed3a8befdbee10d7461e082ecd3e6bab0d24562dc00f53e7eb7bc2b8b75065cf2424456ecb1f9364d74658f28a7b49ffb2c2670ff5442b
-  data.tar.gz: d6eb28d18cdf3aecde015c0da5b39ea529e6118249857e1ea312f4708efb98e3ce7e8a25fae9cc3948870e7afacfafa3e55db46106ae53dd62af8cd261ca1928
+  metadata.gz: 52210c132192267d027e6756e9b34e7871544a968e57627f433d7f5bcef5f8211cd1a548721f6903269397a07c547f65bbd89816a0c4e576e3f9e22a958e777f
+  data.tar.gz: 43132d8f86f99fa6932f479e3bcb0a1c9d224dcd62a2f6b59f18910c9a2ef6d24560d1ad48d35d1c5ce4c7df483ed0d42ebe05663c4d6a94b3899bb0cdb9cda2

data/lib/specwrk/cli.rb

@@ -2,16 +2,90 @@
 
 require "pathname"
 require "securerandom"
+require "json"
 
 require "dry/cli"
 
 require "specwrk"
 require "specwrk/hookable"
+require "specwrk/store"
 
 module Specwrk
   module CLI
     extend Dry::CLI::Registry
 
+    module WorkerProcesses
+      WORKER_INIT_SCRIPT = <<~RUBY
+        writer = IO.for_fd(Integer(ENV.fetch("SPECWRK_FINAL_FD")))
+        $final_output = writer # standard:disable Style/GlobalVars
+        $final_output.sync = true # standard:disable Style/GlobalVars
+        $stdout.sync = true
+        $stderr.sync = true
+
+        require "specwrk/worker"
+
+        trap("INT") do
+          RSpec.world.wants_to_quit = true if defined?(RSpec)
+          exit(1) if Specwrk.force_quit
+          Specwrk.force_quit = true
+        end
+
+        status = Specwrk::Worker.run!
+        $final_output.close # standard:disable Style/GlobalVars
+        exit(status)
+      RUBY
+
+      def start_workers
+        @final_outputs = []
+        @worker_pids = worker_count.times.map do |i|
+          reader, writer = IO.pipe
+          @final_outputs << reader
+
+          env = worker_env_for(i + 1).merge(
+            "SPECWRK_FINAL_FD" => writer.fileno.to_s
+          )
+
+          Process.spawn(
+            env, RbConfig.ruby, "-e", WORKER_INIT_SCRIPT,
+            writer.fileno => writer,
+            :in => :close,
+            :close_others => false
+          ).tap { writer.close }
+        end
+      end
+
+      def drain_outputs
+        @final_outputs.each do |reader|
+          reader.each_line { |line| $stdout.print line }
+          reader.close
+        end
+      end
+
+      def worker_count
+        @worker_count ||= [1, ENV["SPECWRK_COUNT"].to_i].max
+      end
+
+      def worker_env_for(idx)
+        {
+          "TEST_ENV_NUMBER" => idx.to_s,
+          "SPECWRK_FORKED" => idx.to_s,
+          "SPECWRK_ID" => "#{ENV.fetch("SPECWRK_ID", "specwrk-worker")}-#{idx}"
+        }
+      end
+    end
+
+    module PortDiscoverable
+      def find_open_port
+        require "socket"
+
+        server = TCPServer.new("127.0.0.1", 0)
+        port = server.addr[1]
+        server.close
+
+        port
+      end
+    end
+
     module Clientable
       extend Hookable
 
@@ -34,6 +108,7 @@ module Specwrk
 
     module Workable
       extend Hookable
+      include WorkerProcesses
 
       on_included do |base|
         base.unique_option :id, type: :string, desc: "The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used"
@@ -49,44 +124,11 @@ module Specwrk
         ENV["SPECWRK_SEED_WAITS"] = seed_waits.to_s
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
       end
-
-      def start_workers
-        @final_outputs = []
-        @worker_pids = worker_count.times.map do |i|
-          reader, writer = IO.pipe
-          @final_outputs << reader
-
-          Process.fork do
-            ENV["TEST_ENV_NUMBER"] = ENV["SPECWRK_FORKED"] = (i + 1).to_s
-            ENV["SPECWRK_ID"] = ENV["SPECWRK_ID"] + "-#{i + 1}"
-
-            $final_output = writer # standard:disable Style/GlobalVars
-            $final_output.sync = true # standard:disable Style/GlobalVars
-            reader.close
-
-            require "specwrk/worker"
-
-            status = Specwrk::Worker.run!
-            $final_output.close # standard:disable Style/GlobalVars
-            exit(status)
-          end.tap { writer.close }
-        end
-      end
-
-      def drain_outputs
-        @final_outputs.each do |reader|
-          reader.each_line { |line| $stdout.print line }
-          reader.close
-        end
-      end
-
-      def worker_count
-        @worker_count ||= [1, ENV["SPECWRK_COUNT"].to_i].max
-      end
     end
 
     module Servable
       extend Hookable
+      include PortDiscoverable
 
       on_included do |base|
         base.unique_option :port, type: :integer, default: ENV.fetch("SPECWRK_SRV_PORT", "5138"), aliases: ["-p"], desc: "Server port. Overrides SPECWRK_SRV_PORT"
@@ -94,30 +136,22 @@ module Specwrk
         base.unique_option :key, type: :string, aliases: ["-k"], default: ENV.fetch("SPECWRK_SRV_KEY", ""), desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
         base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker or server output is stored. Overrides SPECWRK_OUT"
         base.unique_option :store_uri, type: :string, desc: "Directory where server state is stored. Required for multi-node or multi-process servers."
-        base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWERK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY"
+        base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWRK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWRK_SRV_GROUP_BY"
+        base.unique_option :store_serializer, values: %w[json msgpack], default: ENV.fetch("SPECWRK_STORE_SERIALIZER", "json"), desc: "Serializer to use for store payloads. Overrides SPECWRK_STORE_SERIALIZER"
         base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode"
       end
 
-      on_setup do |port:, bind:, output:, key:, group_by:, verbose:, **opts|
+      on_setup do |port:, bind:, output:, key:, group_by:, verbose:, store_serializer:, **opts|
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
         ENV["SPECWRK_SRV_STORE_URI"] = opts[:store_uri] if opts.key? :store_uri
         ENV["SPECWRK_SRV_VERBOSE"] = "1" if verbose
+        ENV["SPECWRK_STORE_SERIALIZER"] = store_serializer
 
         ENV["SPECWRK_SRV_PORT"] = port
         ENV["SPECWRK_SRV_BIND"] = bind
         ENV["SPECWRK_SRV_KEY"] = key
         ENV["SPECWRK_SRV_GROUP_BY"] = group_by
       end
-
-      def find_open_port
-        require "socket"
-
-        server = TCPServer.new("127.0.0.1", 0)
-        port = server.addr[1]
-        server.close
-
-        port
-      end
     end
 
     class Version < Dry::CLI::Command
@@ -210,6 +244,31 @@ module Specwrk
       include Workable
       include Servable
 
+      SEED_INIT_SCRIPT = <<~'RUBY'
+        require "json"
+        require "specwrk/list_examples"
+        require "specwrk/client"
+
+        def status(msg)
+          print "\e[2K\r#{msg}"
+          $stdout.flush
+        end
+
+        dir = JSON.parse(ENV.fetch("SPECWRK_SEED_DIRS"))
+        max_retries = Integer(ENV.fetch("SPECWRK_MAX_RETRIES", "0"))
+
+        examples = Specwrk::ListExamples.new(dir).examples
+
+        status "Waiting for server to respond..."
+        Specwrk::Client.wait_for_server!
+        status "Server responding ✓"
+        status "Seeding #{examples.length} examples..."
+        Specwrk::Client.new.seed(examples, max_retries)
+        file_count = examples.group_by { |e| e[:file_path] }.keys.size
+        status "🌱 Seeded #{examples.size} examples across #{file_count} files"
+        exit(1) if examples.size.zero?
+      RUBY
+
       desc "Start a server and workers, monitor until complete"
       option :max_retries, default: 0, desc: "Number of times an example will be re-run should it fail"
       argument :dir, type: :array, required: false, desc: "Relative spec directory to run against, default: spec/"
@@ -238,23 +297,7 @@ module Specwrk
         end
 
         return if Specwrk.force_quit
-        seed_pid = Process.fork do
-          require "specwrk/list_examples"
-          require "specwrk/client"
-
-          ENV["SPECWRK_FORKED"] = "1"
-          ENV["SPECWRK_SEED"] = "1"
-          examples = ListExamples.new(dir).examples
-
-          status "Waiting for server to respond..."
-          Client.wait_for_server!
-          status "Server responding ✓"
-          status "Seeding #{examples.length} examples..."
-          Client.new.seed(examples, max_retries)
-          file_count = examples.group_by { |e| e[:file_path] }.keys.size
-          status "🌱 Seeded #{examples.size} examples across #{file_count} files"
-          exit(1) if examples.size.zero?
-        end
+        seed_pid = spawn_seed_process(dir, max_retries)
 
         if Specwrk.wait_for_pids_exit([seed_pid]).value?(1)
           status "Seeding examples failed, exiting."
@@ -279,6 +322,19 @@ module Specwrk
         exit(status)
       end
 
+      def spawn_seed_process(dir, max_retries)
+        Process.spawn(
+          {
+            "SPECWRK_FORKED" => "1",
+            "SPECWRK_SEED" => "1",
+            "SPECWRK_SEED_DIRS" => JSON.dump(dir),
+            "SPECWRK_MAX_RETRIES" => max_retries.to_s
+          },
+          RbConfig.ruby, "-e", SEED_INIT_SCRIPT,
+          close_others: false
+        )
+      end
+
       def status(msg)
         print "\e[2K\r#{msg}"
         $stdout.flush
@@ -286,6 +342,20 @@ module Specwrk
     end
 
     class Watch < Dry::CLI::Command
+      include WorkerProcesses
+      include PortDiscoverable
+
+      SEED_LOOP_INIT_SCRIPT = <<~RUBY
+        require "specwrk/ipc"
+        require "specwrk/seed_loop"
+
+        parent_pid = Integer(ENV.fetch("SPECWRK_IPC_PARENT_PID"))
+        fd = Integer(ENV.fetch("SPECWRK_IPC_FD"))
+        ipc = Specwrk::IPC.from_child_fd(fd, parent_pid: parent_pid)
+
+        Specwrk::SeedLoop.loop!(ipc)
+      RUBY
+
       desc "Start a server and workers, watch for file changes in the current directory, and execute specs"
       option :watchfile, type: :string, default: "Specwrk.watchfile.rb", desc: "Path to watchfile configuration"
       option :count, type: :integer, default: 1, aliases: ["-c"], desc: "The number of worker processes you want to start"
@@ -388,14 +458,19 @@ module Specwrk
         @seed_pid ||= begin
           ipc # must be initialized in the parent process
 
-          @seed_pid = Process.fork do
-            require "specwrk/seed_loop"
-
-            ENV["SPECWRK_FORKED"] = "1"
-            ENV["SPECWRK_SEED"] = "1"
-
-            Specwrk::SeedLoop.loop!(ipc)
-          end
+          ipc.child_socket.close_on_exec = false
+
+          Process.spawn(
+            {
+              "SPECWRK_FORKED" => "1",
+              "SPECWRK_SEED" => "1",
+              "SPECWRK_IPC_FD" => ipc.child_socket.fileno.to_s,
+              "SPECWRK_IPC_PARENT_PID" => Process.pid.to_s
+            },
+            RbConfig.ruby, "-e", SEED_LOOP_INIT_SCRIPT,
+            ipc.child_socket => ipc.child_socket,
+            :close_others => false
+          )
         end
       end
 
@@ -421,50 +496,6 @@ module Specwrk
         print "\e[2K\r#{msg}"
         $stdout.flush
       end
-
-      def start_workers
-        @final_outputs = []
-        @worker_pids = worker_count.times.map do |i|
-          reader, writer = IO.pipe
-          @final_outputs << reader
-
-          Process.fork do
-            ENV["TEST_ENV_NUMBER"] = ENV["SPECWRK_FORKED"] = (i + 1).to_s
-            ENV["SPECWRK_ID"] = "specwrk-worker-#{i + 1}"
-
-            $final_output = writer # standard:disable Style/GlobalVars
-            $final_output.sync = true # standard:disable Style/GlobalVars
-            reader.close
-
-            require "specwrk/worker"
-
-            status = Specwrk::Worker.run!
-            $final_output.close # standard:disable Style/GlobalVars
-            exit(status)
-          end.tap { writer.close }
-        end
-      end
-
-      def drain_outputs
-        @final_outputs.each do |reader|
-          reader.each_line { |line| $stdout.print line }
-          reader.close
-        end
-      end
-
-      def worker_count
-        @worker_count ||= [1, ENV["SPECWRK_COUNT"].to_i].max
-      end
-
-      def find_open_port
-        require "socket"
-
-        server = TCPServer.new("127.0.0.1", 0)
-        port = server.addr[1]
-        server.close
-
-        port
-      end
     end
 
     register "version", Version, aliases: ["v", "-v", "--version"]

data/lib/specwrk/ipc.rb

@@ -2,10 +2,20 @@ require "socket"
 
 module Specwrk
   class IPC
-    def initialize
-      @parent_pid = Process.pid
+    attr_reader :parent_socket, :child_socket
 
-      @parent_socket, @child_socket = UNIXSocket.pair
+    def self.from_child_fd(fd, parent_pid:)
+      new(
+        parent_pid: parent_pid,
+        child_socket: UNIXSocket.for_fd(fd)
+      )
+    end
+
+    def initialize(parent_pid: Process.pid, parent_socket: nil, child_socket: nil)
+      @parent_pid = parent_pid
+
+      @parent_socket, @child_socket = parent_socket, child_socket
+      @parent_socket, @child_socket = UNIXSocket.pair if @parent_socket.nil? && @child_socket.nil?
     end
 
     def write(msg)
@@ -23,7 +33,7 @@ module Specwrk
 
     private
 
-    attr_reader :parent_pid, :parent_socket, :child_socket
+    attr_reader :parent_pid
 
     def socket
       child? ? child_socket : parent_socket

data/lib/specwrk/store/base_adapter.rb

@@ -11,6 +11,14 @@ module Specwrk
         def with_lock(_uri, _key)
           yield
         end
+
+        def serializer
+          @serializer ||= Serializer.resolve
+        end
+
+        def reset_serializer!
+          @serializer = nil
+        end
       end
 
       def initialize(uri, scope)

data/lib/specwrk/store/file_adapter.rb

@@ -3,14 +3,14 @@
 require "json"
 require "base64"
 require "securerandom"
+require "fileutils"
 
 require "specwrk/store/base_adapter"
+require "specwrk/store/serializer"
 
 module Specwrk
   class Store
     class FileAdapter < BaseAdapter
-      EXT = ".wrk.json"
-
       @work_queue = Queue.new
       @threads = []
 
@@ -45,13 +45,17 @@ module Specwrk
             end
           end
         end
+
+        def ext
+          ".wrk.#{serializer.adapter_name}"
+        end
       end
 
       def [](key)
         content = read(key.to_s)
         return unless content
 
-        JSON.parse(content, symbolize_names: true)
+        self.class.serializer.load(content)
       end
 
       def []=(key, value)
@@ -60,7 +64,7 @@ module Specwrk
           delete(key_string)
         else
           filename = filename_for_key(key_string)
-          write(filename, JSON.generate(value))
+          write(filename, self.class.serializer.dump(value))
         end
       end
 
@@ -99,7 +103,7 @@ module Specwrk
           result = result_queue.pop
           next if result.last.nil?
 
-          results[result.first] = JSON.parse(result.last, symbolize_names: true)
+          results[result.first] = self.class.serializer.load(result.last)
         end
 
         read_keys.map { |key| [key.to_s, results[key.to_s]] if results.key?(key.to_s) }.compact.to_h # respect order requested in the returned hash
@@ -110,7 +114,7 @@ module Specwrk
 
         hash_with_filenames = hash.map { |key, value| [key.to_s, [filename_for_key(key.to_s), value]] }.to_h
         hash_with_filenames.each do |key, (filename, value)|
-          content = JSON.generate(value)
+          content = self.class.serializer.dump(value)
 
           self.class.schedule_work do
             result_queue << write(filename, content)
@@ -137,7 +141,7 @@ module Specwrk
 
       def read(key)
         filename = filename_for_key key
-        File.read(filename)
+        File.binread(filename)
       rescue Errno::ENOENT
         nil
       end
@@ -146,7 +150,7 @@ module Specwrk
         File.join(
           path,
           encode_key(key)
-        ) + EXT
+        ) + self.class.ext
       end
 
       def path
@@ -160,7 +164,7 @@ module Specwrk
       end
 
       def decode_key(key)
-        encoded_key_part = File.basename(key).delete_suffix(EXT)
+        encoded_key_part = File.basename(key).delete_suffix(self.class.ext)
         padding_count = (4 - encoded_key_part.length % 4) % 4
 
         Base64.urlsafe_decode64(encoded_key_part + ("=" * padding_count))
@@ -169,7 +173,7 @@ module Specwrk
       def known_key_pairs
         Dir.entries(path).sort.map do |filename|
           next if filename.start_with? "."
-          next unless filename.end_with? EXT
+          next unless filename.end_with? self.class.ext
 
           file_path = File.join(path, filename)
           [decode_key(filename), file_path]

data/lib/specwrk/store/serializer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Specwrk
+  class Store
+    module Serializers
+      module JSON
+        module_function
+
+        def adapter_name
+          "json"
+        end
+
+        def dump(value)
+          ::JSON.generate(value)
+        end
+
+        def load(payload)
+          ::JSON.parse(payload, symbolize_names: true)
+        end
+      end
+
+      module MessagePack
+        module_function
+
+        def ensure_loaded!
+          require "msgpack"
+        rescue LoadError
+          raise LoadError, "Unable to use msgpack, gem not found. Add `gem 'msgpack' to your Gemfile and bundle install"
+        end
+
+        module_function
+
+        def adapter_name
+          "msgpack"
+        end
+
+        def dump(value)
+          ensure_loaded!
+
+          ::MessagePack.dump(value)
+        end
+
+        def load(payload)
+          ensure_loaded!
+
+          ::MessagePack.load(payload, symbolize_keys: true)
+        end
+      end
+    end
+
+    module Serializer
+      module_function
+
+      def resolve(name = ENV.fetch("SPECWRK_STORE_SERIALIZER", "json"))
+        return name if name.respond_to?(:dump) && name.respond_to?(:load)
+
+        case name.to_s.downcase
+        when "", "json"
+          Serializers::JSON
+        when "msgpack", "messagepack"
+          Serializers::MessagePack.tap(&:ensure_loaded!)
+        else
+          raise ArgumentError, "Unsupported serializer #{name.inspect}. Choose json or msgpack."
+        end
+      end
+    end
+  end
+end

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.17.1"
+  VERSION = "0.19.0"
 end

data/lib/specwrk/web/endpoints/complete_and_pop.rb

@@ -9,13 +9,14 @@ module Specwrk
         EXAMPLE_STATUSES = %w[passed failed pending]
 
         def with_response
-          completed.merge!(completed_examples)
-          processing.delete(*(completed_examples.keys + retry_examples.keys))
+          retry_examples # pre-calculate before lock
 
           with_lock do
+            processing.delete(*(completed_examples.keys + retry_examples.keys))
             pending.merge!(retry_examples)
           end
 
+          completed.merge!(completed_examples)
           failure_counts.merge!(retry_examples_new_failure_counts)
 
           update_run_times

data/lib/specwrk/web/endpoints/popable.rb

@@ -17,8 +17,12 @@ module Specwrk
             [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           elsif expired_examples.length.positive?
             expired_examples.each { |_id, example| example[:worker_id] = worker_id }
-            with_lock { pending.push_examples(expired_examples.values) }
-            processing.delete(*expired_examples.keys.map(&:to_s))
+
+            with_lock do
+              pending.push_examples(expired_examples.values)
+              processing.delete(*expired_examples.keys.map(&:to_s))
+            end
+
             @examples = nil
 
             [200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
@@ -29,6 +33,7 @@ module Specwrk
 
         def examples
           @examples ||= begin
+            return [] if pending.empty?
             bucket_id = with_lock { pending.shift_bucket }
             return [] if bucket_id.nil?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.19.0
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -177,6 +177,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 email:
 - daniel@prowestech.com
 executables:
@@ -217,6 +231,7 @@ files:
 - lib/specwrk/store/memory_adapter.rb
 - lib/specwrk/store/pending_store.rb
 - lib/specwrk/store/processing_store.rb
+- lib/specwrk/store/serializer.rb
 - lib/specwrk/store/worker_store.rb
 - lib/specwrk/version.rb
 - lib/specwrk/watcher.rb