specwrk 0.1.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40f4d401d702b91aeb83be0c46ce7ca36cef7c672adf2d5d3b1cdba036121067
-  data.tar.gz: cb7e0992a6003a88a0468f9d08ba90a2922866a36cb53a844740c9409fc9254e
+  metadata.gz: 620355ee669758b42abdd60b638044fbe7bbc4880d51356ea20e9785ab1bb4e3
+  data.tar.gz: 4444c229b60e2b83b5adfc620290965c1a866d5de0cdcfab8a63fab4a7948312
 SHA512:
-  metadata.gz: bfa583f503773db843aba2e962878a00f7cc8c38cfe2cd2c56ab5165394c4a586345347afcb1efef626e920f05fe95afe949f89e1c97a5000d3477ebb0e2bea5
-  data.tar.gz: 53bb263d94bd76886d1800f80a3a9fd8270938c7e794ad983f9c843020e0da9abaeec5c2bb664ba3642b02f2fa2b2c8f8cd537216e5af96b38d302d8cd90538e
+  metadata.gz: ec37f9d67476e6e0ce8ac12d970dc153ca4cdcec1f931b4e35973d71f05459599e8e38b750d8a5bfd81bc218866bc55f1e9dacc561340d9d67fa75b437cd4efa
+  data.tar.gz: 5267ffee5481ca36ce19559e84bc1c276de5351abe755950f22d2b8a2047ba931376d6c07adc69fd678375487d120964c04c090bdb5f121c32cde433d61bfe59

data/README.md

@@ -1,4 +1,165 @@
 # Specwrk
+Run your [RSpec](https://github.com/rspec/rspec) examples across many processors and many nodes for a single build. Or just many processes on a single node. Speeds up your *slow* (minutes/hours not seconds) test suite by running multiple examples in parallel.
+
+One CLI command to:
+
+1. Start a queue server for your current build
+2. Seed the queue server with all possible examples in the current project
+3. Execute
+
+## Install
+Start by adding specwrk to your project or installing it.
+```sh
+$ bundle add specwrk -g development,test
+```
+```sh
+$ gem install specwrk
+```
+
+## CLI
+
+```sh
+$ specwrk --help
+
+Commands:
+  specwrk seed [DIR]            # Seed the server with a list of specs for the run
+  specwrk serve                 # Start a queue server
+  specwrk start [DIR]           # Start a server and workers, monitor until complete
+  specwrk version               # Print version
+  specwrk work                  # Start one or more worker processes
+```
+
+### `specwrk start -c 8 spec/`
+Indended for quick-adhoc runs in development. This command starts a queue server, seeds it with examples from the `spec/` directory, and starts `8` worker processes. It will report the ultimate success or failure.
+
+```sh
+$ start --help
+Command:
+  specwrk start
+
+Usage:
+  specwrk start [DIR]
+
+Description:
+  Start a server and workers, monitor until complete
+
+Arguments:
+  DIR                               # Relative spec directory to run against
+
+Options:
+  --uri=VALUE                       # HTTP URI of the server to pull jobs from. Overrides SPECWRK_SRV_PORT. Default 5138., default: "https://localhost:5138"
+  --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY. Default ''., default: ""
+  --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN. Default main., default: "main"
+  --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT. Default 5., default: "5"
+  --id=VALUE                        # The identifier for this worker. Default specwrk-worker(-COUNT_INDEX)., default: "specwrk-worker"
+  --count=VALUE, -c VALUE           # The number of worker processes you want to start. Default 1., default: 1
+  --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT. Default '.specwrk/'., default: ".specwrk/"
+  --port=VALUE, -p VALUE            # Server port. Overrides SPECWRK_SRV_PORT. Default 5138., default: "5138"
+  --bind=VALUE, -b VALUE            # Server bind address. Overrides SPECWRK_SRV_BIND. Default 127.0.0.1., default: "127.0.0.1"
+  --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY. Default timings.: (file/timings), default: "timings"
+  --[no-]single-run                 # Act on shutdown requests from clients. Default: false., default: false
+  --[no-]verbose                    # Run in verbose mode. Default false., default: false
+  --help, -h                        # Print this help
+```
+
+### `specwrk serve`
+Only start the server process. Intended for use in CI pipelines.
+
+```sh
+$ specwrk serve --help
+Command:
+  specwrk serve
+
+Usage:
+  specwrk serve
+
+Description:
+  Start a queue server
+
+Options:
+  --port=VALUE, -p VALUE            # Server port. Overrides SPECWRK_SRV_PORT. Default 5138., default: "5138"
+  --bind=VALUE, -b VALUE            # Server bind address. Overrides SPECWRK_SRV_BIND. Default 127.0.0.1., default: "127.0.0.1"
+  --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY. Default ''., default: ""
+  --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT. Default '.specwrk/'., default: ".specwrk/"
+  --group-by=VALUE                  # How examples will be grouped for workers; fallback to file if no timings are found. Overrides SPECWERK_SRV_GROUP_BY. Default timings.: (file/timings), default: "timings"
+  --[no-]single-run                 # Act on shutdown requests from clients. Default: false., default: false
+  --[no-]verbose                    # Run in verbose mode. Default false., default: false
+  --help, -h                        # Print this help
+```
+
+### `specwrk seed spec/`
+Seed the configured server with examples from the `spec/` directory. Intended for use in CI pipelines.
+
+```sh
+specwrk seed --help
+Command:
+  specwrk seed
+
+Usage:
+  specwrk seed [DIR]
+
+Description:
+  Seed the server with a list of specs for the run
+
+Arguments:
+  DIR                               # Relative spec directory to run against
+
+Options:
+  --uri=VALUE                       # HTTP URI of the server to pull jobs from. Overrides SPECWRK_SRV_PORT. Default 5138., default: "https://localhost:5138"
+  --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY. Default ''., default: ""
+  --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN. Default main., default: "main"
+  --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT. Default 5., default: "5"
+  --help, -h                        # Print this help
+```
+
+### `specwrk work -c 8`
+Starts `8` worker processes which will pull examples off the seeded server. Intended for use in CI pipelines.
+
+```sh
+$ specwrk work --help
+Command:
+  specwrk work
+
+Usage:
+  specwrk work
+
+Description:
+  Start one or more worker processes
+
+Options:
+  --id=VALUE                        # The identifier for this worker. Default specwrk-worker(-COUNT_INDEX)., default: "specwrk-worker"
+  --count=VALUE, -c VALUE           # The number of worker processes you want to start. Default 1., default: 1
+  --output=VALUE, -o VALUE          # Directory where worker output is stored. Overrides SPECWRK_OUT. Default '.specwrk/'., default: ".specwrk/"
+  --uri=VALUE                       # HTTP URI of the server to pull jobs from. Overrides SPECWRK_SRV_PORT. Default 5138., default: "https://localhost:5138"
+  --key=VALUE, -k VALUE             # Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY. Default ''., default: ""
+  --run=VALUE, -r VALUE             # The run identifier for this job execution. Overrides SPECWRK_RUN. Default main., default: "main"
+  --timeout=VALUE, -t VALUE         # The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT. Default 5., default: "5"
+  --help, -h                        # Print this help
+```
+
+## Configuring your test environment
+If you test suite tracks state, starts servers, etc. and you plan on running many processes on the same node, you'll need to make
+adjustments to avoid conflicting port usage or database/state mutations.
+
+`specwrk` workers will have `TEST_ENV_NUMBER={i}` set to help you configure approriately.
+
+### Rails
+Rails has had easy multi-process test setup for a while now by creating unique test databases per process. For my rails v7.2 app which uses PostgreSQL and Capyabara, I made these changes to my `spec/rails_helper.rb`:
+
+```diff
+++ if ENV["TEST_ENV_NUMBER"]
+++   ActiveRecord::TestDatabases.create_and_load_schema(
+++     ENV["TEST_ENV_NUMBER"].to_i, env_name: ActiveRecord::ConnectionHandling::DEFAULT_ENV.call
+++   )
+++ end
+-- Capybara.server_port = 5550
+++ Capybara.server_port = 5550 + ENV.fetch("TEST_ENV_NUMBER", "1").to_i
+++ Capybara.always_include_port = true
+```
+
+## CI
+1 server N nodes with N processes => 🏎️
+
 TODO!
 
 ## Contributing

data/docker/Dockerfile.server

@@ -0,0 +1,20 @@
+FROM ruby:3.4-alpine
+
+WORKDIR /app
+
+RUN mkdir .specwrk/
+
+ARG SPECWRK_SRV_PORT=5138
+ARG SPECWRK_VERSION=latest
+ARG GEMFILE=specwrk-$SPECWRK_VERSION.gem
+
+COPY $GEMFILE ./
+RUN gem install ./$GEMFILE --no-document
+RUN rm ./$GEMFILE
+
+COPY docker/entrypoint.server.sh /usr/local/bin/entrypoint
+RUN chmod +x /usr/local/bin/entrypoint
+
+EXPOSE ${SPECWRK_SRV_PORT}
+
+ENTRYPOINT ["entrypoint"]

data/docker/entrypoint.server.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+exec specwrk serve --port ${PORT:-$SPECWRK_SRV_PORT} --bind 0.0.0.0 --no-single-run --verbose

data/lib/specwrk/cli.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "pathname"
+require "fileutils"
 
 require "dry/cli"
 
@@ -15,7 +16,7 @@ module Specwrk
       extend Hookable
 
       on_included do |base|
-        base.unique_option :uri, type: :string, default: ENV.fetch("SPECWRK_SRV_URI", "https://localhost:#{ENV.fetch("SPECWRK_SRV_PORT", "5138")}"), desc: "HTTP URI of the server to pull jobs from. Overrides SPECWRK_SRV_PORT. Default 5138."
+        base.unique_option :uri, type: :string, default: ENV.fetch("SPECWRK_SRV_URI", "http://localhost:#{ENV.fetch("SPECWRK_SRV_PORT", "5138")}"), desc: "HTTP URI of the server to pull jobs from. Overrides SPECWRK_SRV_PORT. Default 5138."
         base.unique_option :key, type: :string, default: ENV.fetch("SPECWRK_SRV_KEY", ""), aliases: ["-k"], desc: "Authentication key clients must use for access. Overrides SPECWRK_SRV_KEY. Default ''."
         base.unique_option :run, type: :string, default: ENV.fetch("SPECWRK_RUN", "main"), aliases: ["-r"], desc: "The run identifier for this job execution. Overrides SPECWRK_RUN. Default main."
         base.unique_option :timeout, type: :integer, default: ENV.fetch("SPECWRK_TIMEOUT", "5"), aliases: ["-t"], desc: "The amount of time to wait for the server to respond. Overrides SPECWRK_TIMEOUT. Default 5."
@@ -42,6 +43,7 @@ module Specwrk
         ENV["SPECWRK_ID"] = id
         ENV["SPECWRK_COUNT"] = count.to_s
         ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
+        FileUtils.mkdir_p(ENV["SPECWRK_OUT"])
       end
 
       def start_workers
@@ -52,7 +54,8 @@ module Specwrk
 
             require "specwrk/worker"
 
-            Specwrk::Worker.run!
+            status = Specwrk::Worker.run!
+            exit(status)
           end
         end
       end
@@ -75,10 +78,14 @@ module Specwrk
         base.unique_option :verbose, type: :boolean, default: false, desc: "Run in verbose mode. Default false."
       end
 
-      on_setup do |output:, port:, key:, single_run:, group_by:, verbose:, **|
-        ENV["SPECWRK_SRV_LOG"] = Pathname.new(File.join(output, "server.log")).expand_path(Dir.pwd).to_s if output && !verbose
-        ENV["SPECWRK_SRV_OUTPUT"] = Pathname.new(File.join(output, "report.json")).expand_path(Dir.pwd).to_s if output
+      on_setup do |port:, bind:, output:, key:, single_run:, group_by:, verbose:, **|
+        ENV["SPECWRK_OUT"] = Pathname.new(output).expand_path(Dir.pwd).to_s
+        FileUtils.mkdir_p(ENV["SPECWRK_OUT"])
+
+        ENV["SPECWRK_SRV_LOG"] ||= Pathname.new(File.join(ENV["SPECWRK_OUT"], "server.log")).to_s if output && !verbose
+        ENV["SPECWRK_SRV_OUTPUT"] ||= Pathname.new(File.join(ENV["SPECWRK_OUT"], "report.json")).expand_path(Dir.pwd).to_s if output
         ENV["SPECWRK_SRV_PORT"] = port
+        ENV["SPECWRK_SRV_BIND"] = bind
         ENV["SPECWRK_SRV_KEY"] = key
         ENV["SPECWRK_SRV_SINGLE_RUN"] = "1" if single_run
         ENV["SPECWRK_SRV_GROUP_BY"] = group_by
@@ -127,13 +134,21 @@ module Specwrk
         self.class.setup(**args)
 
         start_workers
-        Process.waitall
+        wait_for_workers_exit
 
         require "specwrk/cli_reporter"
-        status = Specwrk::CLIReporter.new.report
+        Specwrk::CLIReporter.new.report
 
         exit(status)
       end
+
+      def wait_for_workers_exit
+        @exited_pids = Specwrk.wait_for_pids_exit(@worker_pids)
+      end
+
+      def status
+        @exited_pids.value?(1) ? 1 : 0
+      end
     end
 
     class Serve < Dry::CLI::Command
@@ -188,44 +203,24 @@ module Specwrk
           status "Samples seeded ✓"
         end
 
-        wait_for_pids_exit([seed_pid])
+        Specwrk.wait_for_pids_exit([seed_pid])
 
         return if Specwrk.force_quit
         status "Starting #{worker_count} workers..."
         start_workers
 
         status "#{worker_count} workers started ✓\n"
-        wait_for_pids_exit(@worker_pids)
+        Specwrk.wait_for_pids_exit(@worker_pids)
 
         return if Specwrk.force_quit
 
         require "specwrk/cli_reporter"
         status = Specwrk::CLIReporter.new.report
 
-        wait_for_pids_exit([web_pid, seed_pid] + @worker_pids)
+        Specwrk.wait_for_pids_exit([web_pid, seed_pid] + @worker_pids)
         exit(status)
       end
 
-      def wait_for_pids_exit(pids)
-        exited_pids = {}
-
-        loop do
-          pids.each do |pid|
-            next if exited_pids.key? pid
-
-            _, status = Process.waitpid2(pid, Process::WNOHANG)
-            exited_pids[pid] = status.exitstatus if status&.exitstatus
-          rescue Errno::ECHILD
-            exited_pids[pid] = 0
-          end
-
-          break if exited_pids.keys.length == pids.length
-          sleep 0.1
-        end
-
-        exited_pids
-      end
-
       def status(msg)
         print "\e[2K\r#{msg}"
         $stdout.flush

data/lib/specwrk/client.rb

@@ -4,6 +4,12 @@ require "uri"
 require "net/http"
 require "json"
 
+require "specwrk"
+
+# Some rspec setups might use webmock, which intercepts specwrk server calls
+# Let's capture the OG HTTP before that happens
+Specwrk.net_http = Net::HTTP
+
 module Specwrk
   class Client
     def self.connect?
@@ -18,7 +24,8 @@ module Specwrk
 
     def self.build_http
       uri = URI(ENV.fetch("SPECWRK_SRV_URI", "http://localhost:5138"))
-      Net::HTTP.new(uri.host, uri.port).tap do |http|
+      Specwrk.net_http.new(uri.host, uri.port).tap do |http|
+        http.use_ssl = uri.scheme == "https"
         http.open_timeout = ENV.fetch("SPECWRK_TIMEOUT", "5").to_i
         http.read_timeout = ENV.fetch("SPECWRK_TIMEOUT", "5").to_i
         http.keep_alive_timeout = 300
@@ -105,28 +112,28 @@ module Specwrk
     private
 
     def get(path, headers: default_headers, body: nil)
-      request = Net::HTTP::Get.new(path, headers)
+      request = Specwrk.net_http::Get.new(path, headers)
       request.body = body if body
 
       make_request(request)
     end
 
     def post(path, headers: default_headers, body: nil)
-      request = Net::HTTP::Post.new(path, headers)
+      request = Specwrk.net_http::Post.new(path, headers)
       request.body = body if body
 
       make_request(request)
     end
 
     def put(path, headers: default_headers, body: nil)
-      request = Net::HTTP::Put.new(path, headers)
+      request = Specwrk.net_http::Put.new(path, headers)
       request.body = body if body
 
       make_request(request)
     end
 
     def delete(path, headers: default_headers, body: nil)
-      request = Net::HTTP::Delete.new(path, headers)
+      request = Specwrk.net_http::Delete.new(path, headers)
       request.body = body if body
 
       make_request(request)

data/lib/specwrk/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Specwrk
-  VERSION = "0.1.4"
+  VERSION = "0.3.0"
 end

data/lib/specwrk/web/app.rb

@@ -27,7 +27,7 @@ module Specwrk
 
           server_opts = {
             Port: ENV.fetch("SPECWRK_SRV_PORT", "5138").to_i,
-            Host: ENV.fetch("SPECWRK_SRV_BIND", "127.0.0.1"),
+            BindAddress: ENV.fetch("SPECWRK_SRV_BIND", "127.0.0.1"),
             Logger: WEBrick::Log.new($stdout, WEBrick::Log::FATAL),
             AccessLog: [],
             KeepAliveTimeout: 300

data/lib/specwrk/worker/completion_formatter.rb

@@ -5,23 +5,28 @@ module Specwrk
     class CompletionFormatter
       RSpec::Core::Formatters.register self, :stop
 
-      attr_reader :examples
+      attr_reader :examples, :failure
 
       def initialize
         @examples = []
+        @failure = false
       end
 
       def stop(group_notification)
         group_notification.notifications.map do |notification|
+          unless failure
+            @failure = notification.example.execution_result.status == :failed
+          end
+
           examples << {
             id: notification.example.id,
             full_description: notification.example.full_description,
-            status: notification.example.metadata[:execution_result].status,
+            status: notification.example.execution_result.status,
             file_path: notification.example.metadata[:file_path],
             line_number: notification.example.metadata[:line_number],
-            started_at: notification.example.metadata[:execution_result].started_at.iso8601(6),
-            finished_at: notification.example.metadata[:execution_result].finished_at.iso8601(6),
-            run_time: notification.example.metadata[:execution_result].run_time
+            started_at: notification.example.execution_result.started_at.iso8601(6),
+            finished_at: notification.example.execution_result.finished_at.iso8601(6),
+            run_time: notification.example.execution_result.run_time
           }
         end
       end

data/lib/specwrk/worker/executor.rb

@@ -11,6 +11,10 @@ require "specwrk/worker/null_formatter"
 module Specwrk
   class Worker
     class Executor
+      def failure
+        completion_formatter.failure
+      end
+
       def examples
         completion_formatter.examples
       end

data/lib/specwrk/worker.rb

@@ -29,19 +29,27 @@ module Specwrk
         break if Specwrk.force_quit
 
         execute
-      rescue CompletedAllExamplesError, NoMoreExamplesError
-        # TODO: Sleep on NoMoreExamplesError to allow for retries
+      rescue CompletedAllExamplesError
         break
+      rescue NoMoreExamplesError
+        # Wait for the other processes (workers) on the same host to finish
+        # This will cause workers to 'hang' until all work has been completed
+        # TODO: break here if all the other worker processes on this host are done executing examples
+        sleep 0.5
       end
 
       executor.final_output.tap(&:rewind).each_line { |line| $stdout.write line }
 
       @heartbeat_thread.kill
       client.close
+
+      status
     rescue Errno::ECONNREFUSED
       warn "\nServer at #{ENV.fetch("SPECWRK_SRV_URI", "http://localhost:5138")} is refusing connections, exiting..."
+      1
     rescue Errno::ECONNRESET
       warn "\nServer at #{ENV.fetch("SPECWRK_SRV_URI", "http://localhost:5138")} stopped responding to connections, exiting..."
+      1
     end
 
     def execute
@@ -79,6 +87,13 @@ module Specwrk
 
     attr_reader :running, :client, :executor
 
+    def status
+      return 1 if executor.failure
+      return 1 if Specwrk.force_quit
+
+      0
+    end
+
     def warn(msg)
       super("#{ENV.fetch("SPECWRK_ID", "specwrk-worker")}: #{msg}")
     end

data/lib/specwrk.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative "specwrk/version"
+require "specwrk/version"
 
 module Specwrk
   Error = Class.new(StandardError)
@@ -15,7 +15,27 @@ module Specwrk
   @starting_pid = Process.pid
 
   class << self
-    attr_accessor :force_quit
+    attr_accessor :force_quit, :net_http
     attr_reader :starting_pid
+
+    def wait_for_pids_exit(pids)
+      exited_pids = {}
+
+      loop do
+        pids.each do |pid|
+          next if exited_pids.key? pid
+
+          _, status = Process.waitpid2(pid, Process::WNOHANG)
+          exited_pids[pid] = status.exitstatus if status&.exitstatus
+        rescue Errno::ECHILD
+          exited_pids[pid] = 1
+        end
+
+        break if exited_pids.keys.length == pids.length
+        sleep 0.1
+      end
+
+      exited_pids
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: specwrk
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Daniel Westendorf
@@ -121,6 +121,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 email:
 - daniel@prowestech.com
 executables:
@@ -134,6 +148,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- docker/Dockerfile.server
+- docker/entrypoint.server.sh
 - exe/specwrk
 - lib/specwrk.rb
 - lib/specwrk/cli.rb
@@ -155,7 +171,7 @@ files:
 - lib/specwrk/worker/progress_formatter.rb
 homepage: https://github.com/danielwestendorf/specwrk
 licenses:
-- LGPLv3
+- GPL-3.0-or-later
 metadata:
   homepage_uri: https://github.com/danielwestendorf/specwrk
   source_code_uri: https://github.com/danielwestendorf/specwrk