rspec-multiprocess_runner 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8fb56cb7c08de0d53e8894ba0a8354dde4af6a5
-  data.tar.gz: 3012c5d9e218b278b2a11385de30cd38e50a970a
+  metadata.gz: da6e498842698e2a4d310674c27cace4b39ca99d
+  data.tar.gz: dbc698c179206e1233bdb6448bc01611ca5aa036
 SHA512:
-  metadata.gz: 9cb30e3cd5b4dae9599e62e5fbc5c9d9cbf58987df4468fb30357faee82b7bc7bcfa1b72a810de11b9db2c501f92b6a2a9ccf54dfffcf018f23ca08f706a0e25
-  data.tar.gz: 4d952d31b662bf8ef7e93ece5de3bfd171164298ffaca4b176d1f3b7ea954f4e2b205c962c2373de8560217c6e61c7b3a9878da7423ea5a348f13b44cab27911
+  metadata.gz: ef0ef92395a94c984f6f253cddbec507aac1f4f4e0d330009e2a383063fc35be8d3d7463892ad45a8d7c3810b09724de08ec323f4f551ff1641d7070f682c489
+  data.tar.gz: 87582349643f471ba15e90da662ab759539572eebc42f47af8f719b491bac36a2764aaf098da705f18d025f738583dc97bb4493b136d82eeb78a423e4d206383

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+# 1.2.0
+
+* Can now run tests on multiple machines at a time by running in head node mode
+  and others in node mode.  Nodes get which files to run when from the head node
+  in a manner similar to how workers got files from the coordinator (which
+  currently also still happens).
+
+* Uses unencoded TCP, but works with SSH tunnels
+
+* Adds associated command line and rake task options
+
+* Reruns once locally if any files are missing from disconnects
+
 # 1.1.0
 
 * Redo exit codes for arguments, signal hanlding, and exceptions

data/README.md

@@ -1,13 +1,15 @@
 # Rspec::MultiprocessRunner
 
 This gem provides a mechanism for running a suite of RSpec tests in multiple
-processes on the same machine, potentially allowing substantial performance
-improvements.
+processes on the same machine and multiple machines (all reporting back to the
+head node [machine]), potentially allowing substantial performance improvements.
 
-It differs from `parallel_tests` in that it uses a coordinator process to manage
-the workers, hand off work to them, and receive results. This means it can
-dynamically balance the workload among the processors. It also means it can
-provide consolidated results in the console.
+It differs from `parallel_tests` in that it uses a coordinator process on each
+machine to manage the workers, hand off work to them, and receive results. These
+coordinators communicate via simple TCP messages with a head node coordinator to
+remain in sync when running on multiple machines. This means it can dynamically
+balance the workload among the processors and machines. It also means it can
+provide consolidated results in one console.
 
 It follows parallel_tests' environment variable conventions so it's easy to
 use them together. (E.g., parallel_tests has a very nice set of rake tasks
@@ -19,11 +21,14 @@ for setting up parallel environments.)
   time needed to run a suite. Even CPU-bound specs can be aided
 * Provides detailed logging of each example as it completes, including the
   failure message (you don't have to wait until the end to see the failure
-  reason).
+  reason). This is only on the machine running the spec, a final print out does
+  occur on the main machine upon completion, however.
 * Detects, kills, and reports spec files that take longer than expected (five
   minutes by default).
 * Detects and reports spec files that crash (without interrupting the
   remainder of the suite).
+* Head node detects and reruns spec files that are not reported back by the
+  nodes.
 
 ## Limitations
 
@@ -37,6 +42,9 @@ for setting up parallel environments.)
 * Intermediate-quality code. Happy path works, and workers are
   managed/restarted, but:
   * There's no test coverage of the runner itself, only auxiliaries.
+* No security in the TCP messaging, but can work over SSH tunnels. Nodes do
+  verify the file name given against the known files so it will not execute
+  maliciously.
 
 ## Installation
 
@@ -75,6 +83,24 @@ that not that many RSpec options really make sense to pass this way. In
 particular, file selection and output formatting options are unlikely to work
 the way you expect.
 
+Runs as a head node node by default. The following command mimics the defaults:
+
+    $ multirspec -p 2222 -n 5 [Files]
+
+A corresponding node node, for a head node on `head_node.local` would be:
+
+    $ multirspec -H head_node.local -p 2222 -n [Files]
+
+N.B. You must include the same files for the nodes as the head node.
+
+A corresponding set up for a node node using SSH would be:
+
+    $ ssh -nNT -L 2500:localhost:2222 head_node.local &
+    $ multirspec -H localhost -p 2500 -n [Files]
+    $ kill $(jobs -p)
+
+N.B. Ensure that the head node has tcp local port forwarding permitted.
+
 ### Rake
 
 There is a rake task wrapper for `multirspec`:

data/exe/multirspec

@@ -15,7 +15,11 @@ coordinator = RSpec::MultiprocessRunner::Coordinator.new(
     test_env_number_first_is_1: options.first_is_1,
     rspec_options: options.rspec_options,
     log_failing_files: options.log_failing_files,
-    use_given_order: options.use_given_order
+    use_given_order: options.use_given_order,
+    head_node: options.head_node,
+    port: options.port,
+    hostname: options.hostname,
+    max_nodes: options.max_nodes
   }
 )
 

data/lib/rspec/multiprocess_runner/command_line_options.rb

@@ -7,7 +7,7 @@ module RSpec::MultiprocessRunner
   class CommandLineOptions
     attr_accessor :worker_count, :file_timeout_seconds, :example_timeout_seconds,
       :rspec_options, :explicit_files_or_directories, :pattern, :log_failing_files,
-      :first_is_1, :use_given_order
+      :first_is_1, :use_given_order, :port, :head_node, :hostname, :max_nodes
 
     DEFAULT_WORKER_COUNT = 3
 
@@ -20,6 +20,10 @@ module RSpec::MultiprocessRunner
       self.rspec_options = []
       self.first_is_1 = default_first_is_1
       self.use_given_order = false
+      self.port = 2222
+      self.hostname = "localhost"
+      self.head_node = true
+      self.max_nodes = 5
     end
 
     def parse(command_line_args, error_stream=$stderr)
@@ -119,6 +123,22 @@ module RSpec::MultiprocessRunner
           self.use_given_order = true
         end
 
+        parser.on("-p", "--port PORT", Integer, "Communicate using port (#{print_default port})") do |port|
+          self.port = port
+        end
+
+        parser.on("-H", "--hostname HOSTNAME", "Hostname of the head node (#{print_default hostname})") do |hostname|
+          self.hostname = hostname
+        end
+
+        parser.on("-n", "--node", "This node is controlled by a head node") do
+          self.head_node = false
+        end
+
+        parser.on("-m", "--max-nodes MAX_NODES", Integer, "Maximum number of nodes (excluding master) permitted (#{print_default max_nodes})") do |max_nodes|
+          self.max_nodes = max_nodes
+        end
+
         parser.on_tail("-h", "--help", "Prints this help") do
           help_requested!
         end

data/lib/rspec/multiprocess_runner/coordinator.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require 'rspec/multiprocess_runner'
 require 'rspec/multiprocess_runner/worker'
+require 'rspec/multiprocess_runner/file_coordinator'
 
 module RSpec::MultiprocessRunner
   class Coordinator
@@ -11,9 +12,11 @@ module RSpec::MultiprocessRunner
       @test_env_number_first_is_1 = options[:test_env_number_first_is_1]
       @log_failing_files = options[:log_failing_files]
       @rspec_options = options[:rspec_options]
-      @spec_files = options[:use_given_order] ? files : sort_files(files)
+      @file_buffer = []
       @workers = []
       @stopped_workers = []
+      @worker_results = []
+      @file_coordinator = FileCoordinator.new(files, options)
     end
 
     def run
@@ -23,6 +26,12 @@ module RSpec::MultiprocessRunner
       end
       run_loop
       quit_all_workers
+      @file_coordinator.finished
+      if @file_coordinator.missing_files.any?
+        run_loop
+        quit_all_workers
+        @file_coordinator.finished
+      end
       print_summary
 
       exit_code
@@ -36,7 +45,7 @@ module RSpec::MultiprocessRunner
       exit_code = 0
       exit_code |= 1 if any_example_failed?
       exit_code |= 2 if !failed_workers.empty?
-      exit_code |= 4 if !@spec_files.empty?
+      exit_code |= 4 if work_left_to_do? || @file_coordinator.missing_files.any?
       exit_code
     end
 
@@ -72,6 +81,7 @@ module RSpec::MultiprocessRunner
     end
 
     def run_loop
+      add_file_to_buffer
       loop do
         act_on_available_worker_messages(0.3)
         reap_stalled_workers
@@ -104,11 +114,23 @@ module RSpec::MultiprocessRunner
     end
 
     def work_left_to_do?
-      !@spec_files.empty?
+      @file_buffer.any?
+    end
+
+    def add_file_to_buffer
+      file = @file_coordinator.get_file
+      @file_buffer << file if file
+    end
+
+    def get_file
+      if work_left_to_do?
+        add_file_to_buffer
+        @file_buffer.shift
+      end
     end
 
     def failed_workers
-      @stopped_workers.select { |w| w.deactivation_reason }
+      @file_coordinator.failed_workers
     end
 
     def act_on_available_worker_messages(timeout)
@@ -120,12 +142,19 @@ module RSpec::MultiprocessRunner
           if worker_status == :dead
             reap_one_worker(ready_worker, "died")
           elsif work_left_to_do? && !ready_worker.working?
-            ready_worker.run_file(@spec_files.shift)
+            ready_worker.run_file(get_file)
+            send_results(ready_worker)
           end
         end
       end
     end
 
+    def send_results(worker)
+      results_to_send = worker.example_results - @worker_results
+      @worker_results += results_to_send
+      @file_coordinator.send_results(results_to_send)
+    end
+
     def reap_one_worker(worker, reason)
       worker.reap
       worker.deactivation_reason = reason
@@ -135,6 +164,8 @@ module RSpec::MultiprocessRunner
     def mark_worker_as_stopped(worker)
       @stopped_workers << worker
       @workers.reject! { |w| w == worker }
+      send_results(worker)
+      @file_coordinator.send_worker_status(worker) if worker.deactivation_reason
     end
 
     def reap_stalled_workers
@@ -159,7 +190,8 @@ module RSpec::MultiprocessRunner
         )
         @workers << new_worker
         new_worker.start
-        new_worker.run_file(@spec_files.shift)
+        file = get_file
+        new_worker.run_file(file)
       end
     end
 
@@ -182,14 +214,15 @@ module RSpec::MultiprocessRunner
 
     def print_summary
       elapsed = Time.now - @start_time
-      by_status_and_time = combine_example_results.each_with_object({}) do |result, idx|
-        (idx[result.status] ||= []) << result
+      by_status_and_time = combine_example_results.each_with_object(Hash.new { |h, k| h[k] = [] }) do |result, idx|
+        idx[result.status] << result
       end
 
       print_skipped_files_details
       print_pending_example_details(by_status_and_time["pending"])
       print_failed_example_details(by_status_and_time["failed"])
-      log_failed_files(by_status_and_time["failed"]) if @log_failing_files
+      print_missing_files
+      log_failed_files(by_status_and_time["failed"].map(&:file_path).uniq  + @file_coordinator.missing_files.to_a) if @log_failing_files
       print_failed_process_details
       puts
       print_elapsed_time(elapsed)
@@ -198,18 +231,18 @@ module RSpec::MultiprocessRunner
     end
 
     def combine_example_results
-      (@workers + @stopped_workers).flat_map(&:example_results).sort_by { |r| r.time_finished }
+      @file_coordinator.results.sort_by { |r| r.time_finished }
     end
 
     def any_example_failed?
-      (@workers + @stopped_workers).detect { |w| w.example_results.detect { |r| r.status == "failed" } }
+      @file_coordinator.results.detect { |r| r.status == "failed" }
     end
 
     def print_skipped_files_details
-      return if @spec_files.empty?
+      return if !work_left_to_do?
       puts
       puts "Skipped files:"
-      @spec_files.each do |spec_file|
+      @file_coordinator.remaining_files.each do |spec_file|
         puts "  - #{spec_file}"
       end
     end
@@ -235,19 +268,13 @@ module RSpec::MultiprocessRunner
       end
     end
 
-    def log_failed_files(failed_example_results)
-      return if failed_example_results.nil?
-
-      failing_files = Hash.new { |h, k| h[k] = 0 }
-      failed_example_results.each do |failure|
-        failing_files[failure.file_path] += 1
-      end
-
+    def log_failed_files(failed_files)
+      return if failed_files.nil?
       puts
       puts "Writing failures to file: #{@log_failing_files}"
       File.open(@log_failing_files, "w+") do |io|
-        failing_files.each do |(k, _)|
-          io << k
+        failed_files.each do |file|
+          io << file
           io << "\n"
         end
       end
@@ -262,8 +289,9 @@ module RSpec::MultiprocessRunner
       example_count = by_status_and_time.map { |status, results| results.size }.inject(0) { |sum, ct| sum + ct }
       failure_count = by_status_and_time["failed"] ? by_status_and_time["failed"].size : 0
       pending_count = by_status_and_time["pending"] ? by_status_and_time["pending"].size : 0
+      missing_count = @file_coordinator.missing_files.size
       process_failure_count = failed_workers.size
-      skipped_count = @spec_files.size
+      skipped_count = @file_coordinator.remaining_files.size
 
       # Copied from RSpec
       summary = pluralize(example_count, "example")
@@ -271,6 +299,7 @@ module RSpec::MultiprocessRunner
       summary << ", #{pending_count} pending" if pending_count > 0
       summary << ", " << pluralize(process_failure_count, "failed proc") if process_failure_count > 0
       summary << ", " << pluralize(skipped_count, "skipped file") if skipped_count > 0
+      summary << ", " << pluralize(missing_count, "missing file") if missing_count > 0
       puts summary
     end
 
@@ -279,10 +308,17 @@ module RSpec::MultiprocessRunner
       puts
       puts "Failed processes:"
       failed_workers.each do |worker|
-        puts "  - #{worker.pid} (env #{worker.environment_number}) #{worker.deactivation_reason} on #{worker.current_file}"
+        puts "  - #{worker.node}:#{worker.pid} (env #{worker.environment_number}) #{worker.deactivation_reason} on #{worker.current_file}"
       end
     end
 
+    def print_missing_files
+      return if @file_coordinator.missing_files.empty?
+      puts
+      puts "Missing files from disconnects:"
+      @file_coordinator.missing_files.each { |file| puts "  + #{file} was given to a node, which disconnected" }
+    end
+
     def print_elapsed_time(seconds_elapsed)
       minutes = seconds_elapsed.to_i / 60
       seconds = seconds_elapsed % 60

data/lib/rspec/multiprocess_runner/file_coordinator.rb

@@ -0,0 +1,148 @@
+# encoding: utf-8
+require 'rspec/multiprocess_runner'
+require 'socket'
+
+module RSpec::MultiprocessRunner
+  class FileCoordinator
+    attr_reader :results, :failed_workers
+
+    COMMAND_FILE = "file"
+    COMMAND_RESULTS = "results"
+    COMMAND_PROCESS = "process"
+    COMMAND_FINISHED = "finished"
+    COMMAND_START = "start"
+
+    def initialize(files, options={})
+      @spec_files = []
+      @results = Set.new
+      @threads = []
+      @failed_workers = []
+      @spec_files_reference = files.to_set
+      @hostname = options[:hostname]
+      @port = options[:port]
+      @max_threads = options[:max_nodes]
+      @head_node = options[:head_node]
+      if @head_node
+        @spec_files = options[:use_given_order] ? files : sort_files(files)
+        Thread.start { run_tcp_server }
+        @node_socket, head_node_socket = Socket.pair(:UNIX, :STREAM)
+        Thread.start { server_connection_established(head_node_socket) }
+      else
+        count = 100
+        while @node_socket.nil? do
+          begin
+            @node_socket = TCPSocket.new @hostname, @port
+            raise unless start?
+          rescue
+            @node_socket = nil
+            raise if count < 0
+            count -= 1
+            sleep(6)
+          end
+        end
+        puts
+      end
+      ObjectSpace.define_finalizer( self, proc { @node_socket.close } )
+    end
+
+    def remaining_files
+      @spec_files
+    end
+
+    def missing_files
+      if @head_node
+        @spec_files_reference - @results.map(&:file_path) - @failed_workers.map(&:current_file)
+      else
+        []
+      end
+    end
+
+    def get_file
+      begin
+        @node_socket.puts [COMMAND_FILE].to_json
+        file = @node_socket.gets.chomp
+        if @spec_files_reference.include? file
+          return file
+        else
+          return nil # Malformed response, assume done, cease function
+        end
+      rescue
+        return nil # If Error, assume done, cease function
+      end
+    end
+
+    def send_results(results)
+      @node_socket.puts [COMMAND_RESULTS, results].to_json
+    end
+
+    def send_worker_status(worker)
+      @node_socket.puts [COMMAND_PROCESS, worker, Socket.gethostname].to_json
+    end
+
+    def finished
+      if @head_node
+        @tcp_server_running = false
+        @threads.each(&:join)
+        @spec_files += missing_files.to_a
+      else
+        @node_socket.puts [COMMAND_FINISHED].to_json
+      end
+    end
+
+    private
+
+    # Sorting by decreasing size attempts to ensure we don't send the slowest
+    # file to a worker right before all the other workers finish and then end up
+    # waiting for that one process to finish.
+    # In the future it would be nice to log execution time and sort by that.
+    def sort_files(files)
+      # #sort_by caches the File.size result so we only call it once per file.
+      files.sort_by { |file| -File.size(file) }
+    end
+
+    def run_tcp_server
+      server = TCPServer.new @port
+      @tcp_server_running = true
+      ObjectSpace.define_finalizer( self, proc { server.close } )
+      while @threads.size < @max_threads && @tcp_server_running
+        @threads << Thread.start(server.accept) do |client|
+          server_connection_established(client) if @tcp_server_running
+        end
+      end
+    end
+
+    def server_connection_established(socket)
+      loop do
+        raw_response = socket.gets
+        break unless raw_response
+        command, results, node = JSON.parse(raw_response)
+        if command == COMMAND_START
+          socket.puts COMMAND_START
+        elsif command == COMMAND_FILE
+          socket.puts @spec_files.shift
+        elsif command == COMMAND_PROCESS && results
+          @failed_workers << MockWorker.from_json_parse(results, node || "unknown")
+        elsif command == COMMAND_RESULTS && results = results.map { |result|
+          ExampleResult.from_json_parse(result) }
+          @results += results
+        elsif command == COMMAND_FINISHED
+          break
+        end
+      end
+    end
+
+    def work_left_to_do?
+      !@spec_files.empty?
+    end
+
+    def start?
+      begin
+        @node_socket.puts [COMMAND_START].to_json
+        response = @node_socket.gets
+        response && response.chomp == COMMAND_START
+      rescue Errno::EPIPE
+        false
+      end
+    end
+  end
+end

data/lib/rspec/multiprocess_runner/rake_task.rb

@@ -65,6 +65,18 @@ module RSpec::MultiprocessRunner
     # Command line options to pass to the RSpec workers. Defaults to `nil`.
     attr_accessor :rspec_opts
 
+    # Port to use for TCP communication. Defaults to `2222`.
+    attr_accessor :port
+
+    # Be a node to a head node at hostname. Defaults to `false`
+    attr_accessor :node
+
+    # Hostname of head node. Defaults to `localhost`
+    attr_accessor :hostname
+
+    # Max number of connections to head_node. Defaults to `5`
+    attr_accessor :max_nodes
+
     def initialize(*args, &task_block)
       @name            = args.shift || :multispec
       @verbose         = true
@@ -126,6 +138,18 @@ module RSpec::MultiprocessRunner
       if use_given_order
         cmd_parts << '--use-given-order'
       end
+      if port
+        cmd_parts << '--port' << port.to_s
+      end
+      if node
+        cmd_parts << '--node'
+      end
+      if hostname
+        cmd_parts << '--hostname' << hostname
+      end
+      if max_nodes
+        cmd_parts << '--max-nodes' << max_nodes.to_s
+      end
       if files_or_directories
         cmd_parts.concat(files_or_directories)
       end

data/lib/rspec/multiprocess_runner/version.rb

@@ -1,5 +1,5 @@
 module RSpec
   module MultiprocessRunner
-    VERSION = "1.1.0"
+    VERSION = "1.2.0"
   end
 end

data/lib/rspec/multiprocess_runner/worker.rb

@@ -146,6 +146,10 @@ module RSpec::MultiprocessRunner
       act_on_message_from_worker(receive_message_from_worker)
     end
 
+    def to_json(options = nil)
+      { "pid" => @pid, "environment_number" => @environment_number, "current_file" => @current_file, "deactivation_reason" => @deactivation_reason }.to_json
+    end
+
     private
 
     def terminate_then_kill(timeout, message=nil)
@@ -300,16 +304,41 @@ module RSpec::MultiprocessRunner
     end
   end
 
+  class MockWorker
+    attr_reader :pid, :environment_number, :current_file, :deactivation_reason, :node
+
+    def initialize(hash, node)
+      @pid = hash["pid"]
+      @environment_number = hash["environment_number"]
+      @current_file = hash["current_file"]
+      @deactivation_reason = hash["deactivation_reason"]
+      @node = node
+    end
+
+    def self.from_json_parse(hash, node)
+      MockWorker.new(hash, node)
+    end
+  end
+
   # @private
   class ExampleResult
     attr_reader :status, :description, :details, :file_path, :time_finished
 
-    def initialize(example_complete_message)
+    def initialize(example_complete_message, time = Time.now)
+      @hash = example_complete_message
       @status = example_complete_message["example_status"]
       @description = example_complete_message["description"]
       @details = example_complete_message["details"]
       @file_path = example_complete_message["file_path"]
-      @time_finished = Time.now
+      @time_finished = time
+    end
+
+    def to_json(options = nil)
+      { hash: @hash, time: @time_finished.iso8601(9) }.to_json
+    end
+
+    def self.from_json_parse(hash)
+      ExampleResult.new(hash["hash"], Time.iso8601(hash["time"]))
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rspec-multiprocess_runner
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Rhett Sutphin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-08 00:00:00.000000000 Z
+date: 2016-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -105,6 +105,7 @@ files:
 - lib/rspec/multiprocess_runner.rb
 - lib/rspec/multiprocess_runner/command_line_options.rb
 - lib/rspec/multiprocess_runner/coordinator.rb
+- lib/rspec/multiprocess_runner/file_coordinator.rb
 - lib/rspec/multiprocess_runner/rake_task.rb
 - lib/rspec/multiprocess_runner/reporting_formatter.rb
 - lib/rspec/multiprocess_runner/version.rb