specwrk 0.17.1 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93a17f70f0e768222876b22465e7b62d60deb9dbe9761eed9611f31638a7f7cb
-  data.tar.gz: 9137ab50e74ea4e3e46d74496718ae2830af4bb3c1a51e4b0ac2ba53d1691617
+  metadata.gz: fc8e7bf6a7d8bebb0a79a536f403a51fd6cf88948e69e4d03304cd62a039290e
+  data.tar.gz: c0f58e84e321b6407cefb18bad2ad42344f8919ff754471725ab498072e54267
 SHA512:
-  metadata.gz: cbcbb39d8d270fee9fed3a8befdbee10d7461e082ecd3e6bab0d24562dc00f53e7eb7bc2b8b75065cf2424456ecb1f9364d74658f28a7b49ffb2c2670ff5442b
-  data.tar.gz: d6eb28d18cdf3aecde015c0da5b39ea529e6118249857e1ea312f4708efb98e3ce7e8a25fae9cc3948870e7afacfafa3e55db46106ae53dd62af8cd261ca1928
+  metadata.gz: 22fc0af6d23a344521e870194f6ea6c943b313c6596ec3b420a103e533a34ad60fe262b5138b8d0dcdbe1644850cdc633e28510c30d4c0ecea07e9607a4c23e3
+  data.tar.gz: 2a0b62e8140bd41efdbc62838bc8972ce56d9b7f3d7cd316479d5264d16909af2aef9f699d4262cb8dcc2960fdd5ae361a978729403a960b3fd55e0e880060fa

data/lib/specwrk/cli.rb

@@ -2,6 +2,7 @@
 
 require "pathname"
 require "securerandom"
+require "json"
 
 require "dry/cli"
 
@@ -12,6 +13,78 @@ module Specwrk
   module CLI
     extend Dry::CLI::Registry
 
+    module WorkerProcesses
+      WORKER_INIT_SCRIPT = <<~RUBY
+        writer = IO.for_fd(Integer(ENV.fetch("SPECWRK_FINAL_FD")))
+        $final_output = writer # standard:disable Style/GlobalVars
+        $final_output.sync = true # standard:disable Style/GlobalVars
+        $stdout.sync = true
+        $stderr.sync = true
+
+        require "specwrk/worker"
+
+        trap("INT") do
+          RSpec.world.wants_to_quit = true if defined?(RSpec)
+          exit(1) if Specwrk.force_quit
+          Specwrk.force_quit = true
+        end
+
+        status = Specwrk::Worker.run!
+        $final_output.close # standard:disable Style/GlobalVars
+        exit(status)
+      RUBY
+
+      def start_workers
+        @final_outputs = []
+        @worker_pids = worker_count.times.map do |i|
+          reader, writer = IO.pipe
+          @final_outputs << reader
+
+          env = worker_env_for(i + 1).merge(
+            "SPECWRK_FINAL_FD" => writer.fileno.to_s
+          )
+
+          Process.spawn(
+            env, RbConfig.ruby, "-e", WORKER_INIT_SCRIPT,
+            writer.fileno => writer,
+            :in => :close,
+            :close_others => false
+          ).tap { writer.close }
+        end
+      end
+
+      def drain_outputs
+        @final_outputs.each do |reader|
+          reader.each_line { |line| $stdout.print line }
+          reader.close
+        end
+      end
+
+      def worker_count
+        @worker_count ||= [1, ENV["SPECWRK_COUNT"].to_i].max
+      end
+
+      def worker_env_for(idx)
+        {
+          "TEST_ENV_NUMBER" => idx.to_s,
+          "SPECWRK_FORKED" => idx.to_s,
+          "SPECWRK_ID" => "#{ENV.fetch("SPECWRK_ID", "specwrk-worker")}-#{idx}"
+        }
+      end
+    end
+
+    module PortDiscoverable
+      def find_open_port
+        require "socket"
+
+        server = TCPServer.new("127.0.0.1", 0)
+        port = server.addr[1]
+        server.close
+
+        port
+      end
+    end
+
     module Clientable
       extend Hookable
 
@@ -34,6 +107,7 @@ module Specwrk
 
     module Workable
       extend Hookable
+      include WorkerProcesses
 
       on_included do |base|
         base.unique_option :id, type: :string, desc: "The identifier for this worker. Overrides SPECWRK_ID. If none provided one in the format of specwrk-worker-8_RAND_CHARS-COUNT_INDEX will be used"
@@ -49,44 +123,11 @@ module Specwrk
         ENV["SPECWRK_SEED_WAITS"] = seed_waits.to_s
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
       end
-
-      def start_workers
-        @final_outputs = []
-        @worker_pids = worker_count.times.map do |i|
-          reader, writer = IO.pipe
-          @final_outputs << reader
-
-          Process.fork do
-            ENV["TEST_ENV_NUMBER"] = ENV["SPECWRK_FORKED"] = (i + 1).to_s
-            ENV["SPECWRK_ID"] = ENV["SPECWRK_ID"] + "-#{i + 1}"
-
-            $final_output = writer # standard:disable Style/GlobalVars
-            $final_output.sync = true # standard:disable Style/GlobalVars
-            reader.close
-
-            require "specwrk/worker"
-
-            status = Specwrk::Worker.run!
-            $final_output.close # standard:disable Style/GlobalVars
-            exit(status)
-          end.tap { writer.close }
-        end
-      end
-
-      def drain_outputs
-        @final_outputs.each do |reader|
-          reader.each_line { |line| $stdout.print line }
-          reader.close
-        end
-      end
-
-      def worker_count
-        @worker_count ||= [1, ENV["SPECWRK_COUNT"].to_i].max
-      end
     end
 
     module Servable
       extend Hookable
+      include PortDiscoverable
 
       on_included do |base|
         base.unique_option :port, type: :integer, default: ENV.fetch("SPECWRK_SRV_PORT", "5138"), aliases: ["-p"], desc: "Server port. Overrides SPECWRK_SRV_PORT"
@@ -94,7 +135,7 @@ module Specwrk
         base.unique_option :key, type: :string, aliases: ["-k"], default: ENV.fetch("SPECWRK_SRV_KEY", ""), desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY"
         base.unique_option :output, type: :string, default: ENV.fetch("SPECWRK_OUT", ".specwrk/"), aliases: ["-o"], desc: "Directory where worker or server output is stored. Overrides SPECWRK_OUT"
         base.unique_option :store_uri, type: :string, desc: "Directory where server state is stored. Required for multi-node or multi-process servers."
-        base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWERK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY"
+        base.unique_option :group_by, values: %w[file timings], default: ENV.fetch("SPECWRK_SRV_GROUP_BY", "timings"), desc: "How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWRK_SRV_GROUP_BY"
         base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode"
       end
 
@@ -108,16 +149,6 @@ module Specwrk
         ENV["SPECWRK_SRV_KEY"] = key
         ENV["SPECWRK_SRV_GROUP_BY"] = group_by
       end
-
-      def find_open_port
-        require "socket"
-
-        server = TCPServer.new("127.0.0.1", 0)
-        port = server.addr[1]
-        server.close
-
-        port
-      end
     end
 
     class Version < Dry::CLI::Command
@@ -210,6 +241,31 @@ module Specwrk
       include Workable
       include Servable
 
+      SEED_INIT_SCRIPT = <<~'RUBY'
+        require "json"
+        require "specwrk/list_examples"
+        require "specwrk/client"
+
+        def status(msg)
+          print "\e[2K\r#{msg}"
+          $stdout.flush
+        end
+
+        dir = JSON.parse(ENV.fetch("SPECWRK_SEED_DIRS"))
+        max_retries = Integer(ENV.fetch("SPECWRK_MAX_RETRIES", "0"))
+
+        examples = Specwrk::ListExamples.new(dir).examples
+
+        status "Waiting for server to respond..."
+        Specwrk::Client.wait_for_server!
+        status "Server responding ✓"
+        status "Seeding #{examples.length} examples..."
+        Specwrk::Client.new.seed(examples, max_retries)
+        file_count = examples.group_by { |e| e[:file_path] }.keys.size
+        status "🌱 Seeded #{examples.size} examples across #{file_count} files"
+        exit(1) if examples.size.zero?
+      RUBY
+
       desc "Start a server and workers, monitor until complete"
       option :max_retries, default: 0, desc: "Number of times an example will be re-run should it fail"
       argument :dir, type: :array, required: false, desc: "Relative spec directory to run against, default: spec/"
@@ -238,23 +294,7 @@ module Specwrk
         end
 
         return if Specwrk.force_quit
-        seed_pid = Process.fork do
-          require "specwrk/list_examples"
-          require "specwrk/client"
-
-          ENV["SPECWRK_FORKED"] = "1"
-          ENV["SPECWRK_SEED"] = "1"
-          examples = ListExamples.new(dir).examples
-
-          status "Waiting for server to respond..."
-          Client.wait_for_server!
-          status "Server responding ✓"
-          status "Seeding #{examples.length} examples..."
-          Client.new.seed(examples, max_retries)
-          file_count = examples.group_by { |e| e[:file_path] }.keys.size
-          status "🌱 Seeded #{examples.size} examples across #{file_count} files"
-          exit(1) if examples.size.zero?
-        end
+        seed_pid = spawn_seed_process(dir, max_retries)
 
         if Specwrk.wait_for_pids_exit([seed_pid]).value?(1)
           status "Seeding examples failed, exiting."
@@ -279,6 +319,19 @@ module Specwrk
         exit(status)
       end
 
+      def spawn_seed_process(dir, max_retries)
+        Process.spawn(
+          {
+            "SPECWRK_FORKED" => "1",
+            "SPECWRK_SEED" => "1",
+            "SPECWRK_SEED_DIRS" => JSON.dump(dir),
+            "SPECWRK_MAX_RETRIES" => max_retries.to_s
+          },
+          RbConfig.ruby, "-e", SEED_INIT_SCRIPT,
+          close_others: false
+        )
+      end
+
       def status(msg)
         print "\e[2K\r#{msg}"
         $stdout.flush
@@ -286,6 +339,20 @@ module Specwrk
     end
 
     class Watch < Dry::CLI::Command
+      include WorkerProcesses
+      include PortDiscoverable
+
+      SEED_LOOP_INIT_SCRIPT = <<~RUBY
+        require "specwrk/ipc"
+        require "specwrk/seed_loop"
+
+        parent_pid = Integer(ENV.fetch("SPECWRK_IPC_PARENT_PID"))
+        fd = Integer(ENV.fetch("SPECWRK_IPC_FD"))
+        ipc = Specwrk::IPC.from_child_fd(fd, parent_pid: parent_pid)
+
+        Specwrk::SeedLoop.loop!(ipc)
+      RUBY
+
       desc "Start a server and workers, watch for file changes in the current directory, and execute specs"
       option :watchfile, type: :string, default: "Specwrk.watchfile.rb", desc: "Path to watchfile configuration"
       option :count, type: :integer, default: 1, aliases: ["-c"], desc: "The number of worker processes you want to start"
@@ -388,14 +455,19 @@ module Specwrk
         @seed_pid ||= begin
           ipc # must be initialized in the parent process
 
-          @seed_pid = Process.fork do
-            require "specwrk/seed_loop"
-
-            ENV["SPECWRK_FORKED"] = "1"
-            ENV["SPECWRK_SEED"] = "1"
-
-            Specwrk::SeedLoop.loop!(ipc)
-          end
+          ipc.child_socket.close_on_exec = false
+
+          Process.spawn(
+            {
+              "SPECWRK_FORKED" => "1",
+              "SPECWRK_SEED" => "1",
+              "SPECWRK_IPC_FD" => ipc.child_socket.fileno.to_s,
+              "SPECWRK_IPC_PARENT_PID" => Process.pid.to_s
+            },
+            RbConfig.ruby, "-e", SEED_LOOP_INIT_SCRIPT,
+            ipc.child_socket => ipc.child_socket,
+            :close_others => false
+          )
         end
       end
 
@@ -421,50 +493,6 @@ module Specwrk
         print "\e[2K\r#{msg}"
         $stdout.flush
       end
-
-      def start_workers
-        @final_outputs = []
-        @worker_pids = worker_count.times.map do |i|
-          reader, writer = IO.pipe
-          @final_outputs << reader
-
-          Process.fork do
-            ENV["TEST_ENV_NUMBER"] = ENV["SPECWRK_FORKED"] = (i + 1).to_s
-            ENV["SPECWRK_ID"] = "specwrk-worker-#{i + 1}"
-
-            $final_output = writer # standard:disable Style/GlobalVars
-            $final_output.sync = true # standard:disable Style/GlobalVars
-            reader.close
-
-            require "specwrk/worker"
-
-            status = Specwrk::Worker.run!
-            $final_output.close # standard:disable Style/GlobalVars
-            exit(status)
-          end.tap { writer.close }
-        end
-      end
-
-      def drain_outputs
-        @final_outputs.each do |reader|
-          reader.each_line { |line| $stdout.print line }
-          reader.close
-        end
-      end
-
-      def worker_count
-        @worker_count ||= [1, ENV["SPECWRK_COUNT"].to_i].max
-      end
-
-      def find_open_port
-        require "socket"
-
-        server = TCPServer.new("127.0.0.1", 0)
-        port = server.addr[1]
-        server.close
-
-        port
-      end
     end
 
     register "version", Version, aliases: ["v", "-v", "--version"]

data/lib/specwrk/ipc.rb

@@ -2,10 +2,20 @@ require "socket"
 
 module Specwrk
   class IPC
-    def initialize
-      @parent_pid = Process.pid
+    attr_reader :parent_socket, :child_socket
 
-      @parent_socket, @child_socket = UNIXSocket.pair
+    def self.from_child_fd(fd, parent_pid:)
+      new(
+        parent_pid: parent_pid,
+        child_socket: UNIXSocket.for_fd(fd)
+      )
+    end
+
+    def initialize(parent_pid: Process.pid, parent_socket: nil, child_socket: nil)
+      @parent_pid = parent_pid
+
+      @parent_socket, @child_socket = parent_socket, child_socket
+      @parent_socket, @child_socket = UNIXSocket.pair if @parent_socket.nil? && @child_socket.nil?
     end
 
     def write(msg)
@@ -23,7 +33,7 @@ module Specwrk
 
     private
 
-    attr_reader :parent_pid, :parent_socket, :child_socket
+    attr_reader :parent_pid
 
     def socket
       child? ? child_socket : parent_socket

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.17.1"
+  VERSION = "0.18.0"
 end

data/lib/specwrk/web/endpoints/complete_and_pop.rb

@@ -9,13 +9,14 @@ module Specwrk
         EXAMPLE_STATUSES = %w[passed failed pending]
 
         def with_response
-          completed.merge!(completed_examples)
-          processing.delete(*(completed_examples.keys + retry_examples.keys))
+          retry_examples # pre-calculate before lock
 
           with_lock do
+            processing.delete(*(completed_examples.keys + retry_examples.keys))
             pending.merge!(retry_examples)
           end
 
+          completed.merge!(completed_examples)
           failure_counts.merge!(retry_examples_new_failure_counts)
 
           update_run_times

data/lib/specwrk/web/endpoints/popable.rb

@@ -17,8 +17,12 @@ module Specwrk
             [410, {"content-type" => "text/plain"}, ["That's a good lad. Run along now and go home."]]
           elsif expired_examples.length.positive?
             expired_examples.each { |_id, example| example[:worker_id] = worker_id }
-            with_lock { pending.push_examples(expired_examples.values) }
-            processing.delete(*expired_examples.keys.map(&:to_s))
+
+            with_lock do
+              pending.push_examples(expired_examples.values)
+              processing.delete(*expired_examples.keys.map(&:to_s))
+            end
+
             @examples = nil
 
             [200, {"content-type" => "application/json"}, [JSON.generate(examples)]]
@@ -29,6 +33,7 @@ module Specwrk
 
         def examples
           @examples ||= begin
+            return [] if pending.empty?
             bucket_id = with_lock { pending.shift_bucket }
             return [] if bucket_id.nil?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.18.0
 platform: ruby
 authors:
 - Daniel Westendorf