turbo_tests 1.2.0 → 1.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae7178127f711a2ea749e60ff16ed6e3f46011737c02bf1fd2912b6777869b0a
-  data.tar.gz: d07339025c120451ca4e30fd13da954ded64234081759c0b0e9562954447462a
+  metadata.gz: b8a0dc37755b687ea503d265d864eb959d2bda3668e3f21c4b09af48e6fb852a
+  data.tar.gz: 8399b27819ae06f123b0695754fe074cb8819d385cd2a4d932c12b9f30a88124
 SHA512:
-  metadata.gz: cbb71d599830fa9e86287f389aca4c678794b2258d287dab88bd975f7648bbb4e78b9986c24274b231730d069c88df0d3ce81ce46797723ae44248dec056355f
-  data.tar.gz: 12df6012daff0f3925aab5204247174d07a6d41d85b6c3a3a79f6f9a79cdc1a7d3c0b4e874cdf405024913d856683539e31257ab4863883f57af77acbda23154
+  metadata.gz: cae6d818935454f40984af484fa8d87763732b578a4bb901679434c40a0d5dc152c671726523877247375c4572849c907e4190faab6e0681abb3b2d8db50e1bd
+  data.tar.gz: 41a44281ddb35a3440eaa633bafecd7e9bb585acf132e69180a521e68148a9feb10121ac1a7bd38c38892d56c0ba326215bacd590380355ece8b02715667697b

data/.github/workflows/snyk_ruby-analysis.yml

@@ -0,0 +1,33 @@
+# A sample workflow which checks out your Infrastructure as Code Configuration files,
+# such as Kubernetes, Helm & Terraform and scans them for any security issues.
+# The results are then uploaded to GitHub Security Code Scanning
+#
+# For more examples, including how to limit scans to only high-severity issues
+# and fail PR checks, see https://github.com/snyk/actions/
+
+name: Snyk Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '0 0 * * 0'
+
+jobs:
+  snyk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run Snyk to check configuration files for security issues
+        # Snyk can be used to break the build when it detects security issues.
+        # In this case we want to upload the issues to GitHub Code Scanning
+        continue-on-error: true
+        uses: snyk/actions/ruby@master
+        env:
+          # In order to use the Snyk Action you will need to have a Snyk API token.
+          # More details in https://github.com/snyk/actions#getting-your-snyk-token
+          # or you can signup for free at https://snyk.io/login
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

data/.gitignore

@@ -9,3 +9,60 @@
 
 # rspec failure tracking
 .rspec_status
+
+
+# Created by https://www.toptal.com/developers/gitignore/api/macos,vim
+# Edit at https://www.toptal.com/developers/gitignore?templates=macos,vim
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/macos,vim
+

data/.rspec

@@ -1,3 +1,4 @@
 --format documentation
 --color
+--tty
 --require spec_helper

data/Gemfile

@@ -3,4 +3,4 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in turbo_tests.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    turbo_tests (1.2.0)
+    turbo_tests (1.2.5)
       bundler
       parallel_tests (~> 3.3)
-      rspec (~> 3.10.0)
+      rspec (~> 3.10)
 
 GEM
   remote: https://rubygems.org/
@@ -13,32 +13,32 @@ GEM
     diff-lcs (1.4.4)
     method_source (1.0.0)
     parallel (1.20.1)
-    parallel_tests (3.4.0)
+    parallel_tests (3.7.0)
       parallel
-    pry (0.13.1)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rake (12.3.3)
+    rake (13.0.6)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
       rspec-expectations (~> 3.10.0)
       rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.0)
+    rspec-core (3.10.1)
       rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.0)
+    rspec-support (3.10.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  pry (~> 0.13)
-  rake (~> 12.0)
+  pry (~> 0.14)
+  rake (~> 13.0)
   turbo_tests!
 
 BUNDLED WITH

data/README.md

@@ -1,3 +1,5 @@
+![Tests](https://github.com/serpapi/turbo_tests/workflows/Tests/badge.svg)
+
 # TurboTests
 
 Runner for [`grosser/parallel_tests`](https://github.com/grosser/parallel_tests) with incremental summarized output. Based on [Discourse](https://github.com/discourse/discourse/blob/6b9784cf8a18636bce281a7e4d18e65a0cbc6290/lib/turbo_tests.rb) and [RubyGems](https://github.com/rubygems/rubygems/tree/390335ceb351668cd433bd5bb9823dd021f82533/bundler/tool) work in this area.
@@ -11,23 +13,17 @@ This feature [doesn't fit vision of `parallel_tests` author](https://github.com/
 ```bash
 
 $ bundle exec rake parallel_tests:spec[^spec/search]
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
+.................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
 
 Finished in 1 minute 6.92 seconds (files took 6.95 seconds to load)
 2616 examples, 0 failures
 
-.........................................................................................................................................F.................................................
-.......................................................................................F...................................................................................................
-...........................................................................................................................................................................................
+.........................................................................................................................................F........................................................................................................................................F..............................................................................................................................................................................................................................................................................................
 
 Finished in 1 minute 35.05 seconds (files took 6.26 seconds to load)
 2158 examples, 2 failures
 
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
+.................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
 
 Finished in 1 minute 35.05 seconds (files took 6.26 seconds to load)
 2158 examples, 0 failures
@@ -37,15 +33,7 @@ Finished in 1 minute 35.05 seconds (files took 6.26 seconds to load)
 
 ```bash
 $ bundle exec turbo_tests
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-.........................................................................................................................................F.................................................
-.......................................................................................F...................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
+..........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................F........................................................................................................................................F..............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
 
 Finished in 2 minute 25.15 seconds (files took 0 seconds to load)
 6873 examples, 2 failures

data/fixtures/rspec/errors_outside_of_examples_spec.rb

@@ -0,0 +1,5 @@
+RSpec.describe "Fixture of spec file with errors outside of examples" do
+  it("passes") { expect(2 * 2).to eql(4) }
+
+  1 / 0
+end

data/fixtures/rspec/pending_exceptions_spec.rb

@@ -0,0 +1,15 @@
+RSpec.describe "Fixture of spec file with pending failed examples" do
+  it "is implemented but skipped with 'pending'" do
+    pending("TODO: skipped with 'pending'")
+
+    expect(2).to eq(3)
+  end
+
+  it "is implemented but skipped with 'skip'", skip: "TODO: skipped with 'skip'" do
+    expect(100).to eq(500)
+  end
+
+  xit "is implemented but skipped with 'xit'" do
+    expect(1).to eq(42)
+  end
+end

data/lib/turbo_tests.rb

@@ -23,28 +23,29 @@ module TurboTests
         klass =
           Class.new(FakeException) {
             define_singleton_method(:name) do
-              obj["class_name"]
+              obj[:class_name]
             end
           }
 
         klass.new(
-          obj["backtrace"],
-          obj["message"],
-          FakeException.from_obj(obj["cause"])
+          obj[:backtrace],
+          obj[:message],
+          FakeException.from_obj(obj[:cause])
         )
       end
     end
   end
 
-  FakeExecutionResult = Struct.new(:example_skipped?, :pending_message, :status, :pending_fixed?, :exception)
+  FakeExecutionResult = Struct.new(:example_skipped?, :pending_message, :status, :pending_fixed?, :exception, :pending_exception)
   class FakeExecutionResult
     def self.from_obj(obj)
       new(
-        obj["example_skipped?"],
-        obj["pending_message"],
-        obj["status"].to_sym,
-        obj["pending_fixed?"],
-        FakeException.from_obj(obj["exception"])
+        obj[:example_skipped?],
+        obj[:pending_message],
+        obj[:status].to_sym,
+        obj[:pending_fixed?],
+        FakeException.from_obj(obj[:exception]),
+        FakeException.from_obj(obj[:exception])
       )
     end
   end
@@ -52,24 +53,24 @@ module TurboTests
   FakeExample = Struct.new(:execution_result, :location, :description, :full_description, :metadata, :location_rerun_argument)
   class FakeExample
     def self.from_obj(obj)
-      metadata = obj["metadata"]
+      metadata = obj[:metadata]
 
-      metadata["shared_group_inclusion_backtrace"].map! do |frame|
+      metadata[:shared_group_inclusion_backtrace].map! do |frame|
         RSpec::Core::SharedExampleGroupInclusionStackFrame.new(
-          frame["shared_group_name"],
-          frame["inclusion_location"]
+          frame[:shared_group_name],
+          frame[:inclusion_location]
         )
       end
 
-      metadata[:shared_group_inclusion_backtrace] = metadata.delete("shared_group_inclusion_backtrace")
+      metadata[:shared_group_inclusion_backtrace] = metadata.delete(:shared_group_inclusion_backtrace)
 
       new(
-        FakeExecutionResult.from_obj(obj["execution_result"]),
-        obj["location"],
-        obj["description"],
-        obj["full_description"],
+        FakeExecutionResult.from_obj(obj[:execution_result]),
+        obj[:location],
+        obj[:description],
+        obj[:full_description],
         metadata,
-        obj["location_rerun_argument"]
+        obj[:location_rerun_argument]
       )
     end
 

data/lib/turbo_tests/cli.rb

@@ -17,7 +17,7 @@ module TurboTests
       fail_fast = nil
 
       OptionParser.new { |opts|
-        opts.banner = <<-BANNER.gsub(/^          /, "")
+        opts.banner = <<~BANNER
           Run all tests in parallel, giving each process ENV['TEST_ENV_NUMBER'] ('1', '2', '3', ...).
 
           Uses parallel_tests under the hood, but reports test results incrementally. Based on Discourse and RubyGems work in this area.

data/lib/turbo_tests/json_rows_formatter.rb

@@ -22,10 +22,14 @@ module TurboTests
   class JsonRowsFormatter
     RSpec::Core::Formatters.register(
       self,
+      :start,
       :close,
       :example_failed,
       :example_passed,
       :example_pending,
+      :example_group_started,
+      :example_group_finished,
+      :message,
       :seed
     )
 
@@ -35,37 +39,65 @@ module TurboTests
       @output = output
     end
 
+    def start(notification)
+      output_row(
+        type: :load_summary,
+        summary: load_summary_to_json(notification)
+      )
+    end
+
+    def example_group_started(notification)
+      output_row(
+        type: :group_started,
+        group: group_to_json(notification)
+      )
+    end
+
+    def example_group_finished(notification)
+      output_row(
+        type: :group_finished,
+        group: group_to_json(notification)
+      )
+    end
+
     def example_passed(notification)
       output_row(
-        "type" => :example_passed,
-        "example" => example_to_json(notification.example)
+        type: :example_passed,
+        example: example_to_json(notification.example)
       )
     end
 
     def example_pending(notification)
       output_row(
-        "type" => :example_pending,
-        "example" => example_to_json(notification.example)
+        type: :example_pending,
+        example: example_to_json(notification.example)
       )
     end
 
     def example_failed(notification)
       output_row(
-        "type" => :example_failed,
-        "example" => example_to_json(notification.example)
+        type: :example_failed,
+        example: example_to_json(notification.example)
       )
     end
 
     def seed(notification)
       output_row(
-        "type" => :seed,
-        "seed" => notification.seed
+        type: :seed,
+        seed: notification.seed
       )
     end
 
     def close(notification)
       output_row(
-        "type" => :close
+        type: :close
+      )
+    end
+
+    def message(notification)
+      output_row(
+        type: :message,
+        message: notification.message
       )
     end
 
@@ -74,47 +106,65 @@ module TurboTests
     def exception_to_json(exception)
       if exception
         {
-          "class_name" => exception.class.name.to_s,
-          "backtrace" => exception.backtrace,
-          "message" => exception.message,
-          "cause" => exception_to_json(exception.cause)
+          class_name: exception.class.name.to_s,
+          backtrace: exception.backtrace,
+          message: exception.message,
+          cause: exception_to_json(exception.cause)
         }
       end
     end
 
     def execution_result_to_json(result)
       {
-        "example_skipped?" => result.example_skipped?,
-        "pending_message" => result.pending_message,
-        "status" => result.status,
-        "pending_fixed?" => result.pending_fixed?,
-        "exception" => exception_to_json(result.exception)
+        example_skipped?: result.example_skipped?,
+        pending_message: result.pending_message,
+        status: result.status,
+        pending_fixed?: result.pending_fixed?,
+        exception: exception_to_json(result.exception || result.pending_exception)
       }
     end
 
     def stack_frame_to_json(frame)
       {
-        "shared_group_name" => frame.shared_group_name,
-        "inclusion_location" => frame.inclusion_location
+        shared_group_name: frame.shared_group_name,
+        inclusion_location: frame.inclusion_location
       }
     end
 
     def example_to_json(example)
       {
-        "execution_result" => execution_result_to_json(example.execution_result),
-        "location" => example.location,
-        "description" => example.description,
-        "full_description" => example.full_description,
-        "metadata" => {
-          "shared_group_inclusion_backtrace" =>
-            example.metadata[:shared_group_inclusion_backtrace].map { |frame| stack_frame_to_json(frame) }
+        execution_result: execution_result_to_json(example.execution_result),
+        location: example.location,
+        description: example.description,
+        full_description: example.full_description,
+        metadata: {
+          shared_group_inclusion_backtrace:
+            example
+              .metadata[:shared_group_inclusion_backtrace]
+              .map { |frame| stack_frame_to_json(frame) }
         },
-        "location_rerun_argument" => example.location_rerun_argument
+        location_rerun_argument: example.location_rerun_argument
+      }
+    end
+
+    def load_summary_to_json(notification)
+      {
+        count: notification.count,
+        load_time: notification.load_time,
+      }
+    end
+
+    def group_to_json(notification)
+      {
+        group: {
+          description: notification.group.description
+        }
       }
     end
 
     def output_row(obj)
-      output.puts ENV["RSPEC_FORMATTER_OUTPUT_ID"] + obj.to_json
+      output.puts(obj.to_json)
+      output.flush
     end
   end
 end

data/lib/turbo_tests/reporter.rb

@@ -2,6 +2,8 @@
 
 module TurboTests
   class Reporter
+    attr_writer :load_time
+
     def self.from_config(formatter_config, start_time)
       reporter = new(start_time)
 
@@ -9,7 +11,7 @@ module TurboTests
         name, outputs = config.values_at(:name, :outputs)
 
         outputs.map! do |filename|
-          filename == "-" ? STDOUT : File.open(filename, "w")
+          filename == "-" ? $stdout : File.open(filename, "w")
         end
 
         reporter.add(name, outputs)
@@ -26,7 +28,10 @@ module TurboTests
       @pending_examples = []
       @failed_examples = []
       @all_examples = []
+      @messages = []
       @start_time = start_time
+      @load_time = 0
+      @errors_outside_of_examples_count = 0
     end
 
     def add(name, outputs)
@@ -45,6 +50,14 @@ module TurboTests
       end
     end
 
+    def group_started(notification)
+      delegate_to_formatters(:example_group_started, notification)
+    end
+
+    def group_finished
+      delegate_to_formatters(:example_group_finished, nil)
+    end
+
     def example_passed(example)
       delegate_to_formatters(:example_passed, example.notification)
 
@@ -65,8 +78,18 @@ module TurboTests
       @failed_examples << example
     end
 
+    def message(message)
+      delegate_to_formatters(:message, RSpec::Core::Notifications::MessageNotification.new(message))
+      @messages << message
+    end
+
+    def error_outside_of_examples
+      @errors_outside_of_examples_count += 1
+    end
+
     def finish
-      end_time = Time.now
+      # SEE: https://bit.ly/2NP87Cz
+      end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
       delegate_to_formatters(:start_dump,
         RSpec::Core::Notifications::NullNotification)
@@ -84,8 +107,8 @@ module TurboTests
           @all_examples,
           @failed_examples,
           @pending_examples,
-          0,
-          0
+          @load_time,
+          @errors_outside_of_examples_count
         ))
       delegate_to_formatters(:close,
         RSpec::Core::Notifications::NullNotification)

data/lib/turbo_tests/runner.rb

@@ -3,17 +3,27 @@
 require "json"
 require "parallel_tests/rspec/runner"
 
+require_relative "../utils/hash_extension"
+
 module TurboTests
   class Runner
+    using CoreExtensions
+
     def self.run(opts = {})
       files = opts[:files]
       formatters = opts[:formatters]
       tags = opts[:tags]
-      start_time = opts.fetch(:start_time) { Time.now }
+
+      # SEE: https://bit.ly/2NP87Cz
+      start_time = opts.fetch(:start_time) { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
       verbose = opts.fetch(:verbose, false)
       fail_fast = opts.fetch(:fail_fast, nil)
       count = opts.fetch(:count, nil)
 
+      if verbose
+        STDERR.puts "VERBOSE"
+      end
+
       reporter = Reporter.from_config(formatters, start_time)
 
       new(
@@ -33,12 +43,13 @@ module TurboTests
       @verbose = opts[:verbose]
       @fail_fast = opts[:fail_fast]
       @count = opts[:count]
-
+      @load_time = 0
+      @load_count = 0
       @failure_count = 0
-      @runtime_log = "tmp/parallel_runtime_rspec.log"
 
       @messages = Queue.new
       @threads = []
+      @error = false
     end
 
     def run
@@ -47,17 +58,33 @@ module TurboTests
         ParallelTests::RSpec::Runner.tests_with_size(@files, {}).size
       ].min
 
+      use_runtime_info = @files == ["spec"]
+
+      group_opts = {}
+
+      if use_runtime_info
+        group_opts[:runtime_log] = "tmp/turbo_rspec_runtime.log"
+      else
+        group_opts[:group_by] = :filesize
+      end
+
       tests_in_groups =
         ParallelTests::RSpec::Runner.tests_in_groups(
           @files,
           @num_processes,
-          runtime_log: @runtime_log
+          **group_opts
         )
 
+      setup_tmp_dir
+
+      subprocess_opts = {
+        record_runtime: use_runtime_info
+      }
+
       report_number_of_tests(tests_in_groups)
 
-      tests_in_groups.each_with_index do |tests, process_id|
-        start_regular_subprocess(tests, process_id + 1)
+      wait_threads = tests_in_groups.map.with_index do |tests, process_id|
+        start_regular_subprocess(tests, process_id + 1, **subprocess_opts)
       end
 
       handle_messages
@@ -66,38 +93,64 @@ module TurboTests
 
       @threads.each(&:join)
 
-      @reporter.failed_examples.empty?
+      @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?)
     end
 
-    protected
+    private
+
+    def setup_tmp_dir
+      begin
+        FileUtils.rm_r("tmp/test-pipes")
+      rescue Errno::ENOENT
+      end
 
-    def start_regular_subprocess(tests, process_id)
+      FileUtils.mkdir_p("tmp/test-pipes/")
+    end
+
+    def start_regular_subprocess(tests, process_id, **opts)
       start_subprocess(
         {"TEST_ENV_NUMBER" => process_id.to_s},
         @tags.map { |tag| "--tag=#{tag}" },
         tests,
-        process_id
+        process_id,
+        **opts
       )
     end
 
-    def start_subprocess(env, extra_args, tests, process_id)
+    def start_subprocess(env, extra_args, tests, process_id, record_runtime:)
       if tests.empty?
         @messages << {
-          "type" => "exit",
-          "process_id" => process_id
+          type: "exit",
+          process_id: process_id
         }
       else
-        require "securerandom"
-        env["RSPEC_FORMATTER_OUTPUT_ID"] = SecureRandom.uuid
-        env["RUBYOPT"] = "-I#{File.expand_path("..", __dir__)}"
+        tmp_filename = "tmp/test-pipes/subprocess-#{process_id}"
+
+        begin
+          File.mkfifo(tmp_filename)
+        rescue Errno::EEXIST
+        end
+
+        env["RUBYOPT"] = ["-I#{File.expand_path("..", __dir__)}", ENV["RUBYOPT"]].compact.join(" ")
+        env["RSPEC_SILENCE_FILTER_ANNOUNCEMENTS"] = "1"
+
+        record_runtime_options =
+          if record_runtime
+            [
+              "--format", "ParallelTests::RSpec::RuntimeLogger",
+              "--out", "tmp/turbo_rspec_runtime.log",
+            ]
+          else
+            []
+          end
 
         command = [
           ENV["BUNDLE_BIN_PATH"], "exec", "rspec",
           *extra_args,
-          "--seed", rand(2**16).to_s,
-          "--format", "ParallelTests::RSpec::RuntimeLogger",
-          "--out", @runtime_log,
+          "--seed", rand(0xFFFF).to_s,
           "--format", "TurboTests::JsonRowsFormatter",
+          "--out", tmp_filename,
+          *record_runtime_options,
           *tests
         ]
 
@@ -110,29 +163,33 @@ module TurboTests
           STDERR.puts "Process #{process_id}: #{command_str}"
         end
 
-        _stdin, stdout, stderr, _wait_thr = Open3.popen3(env, *command)
+        stdin, stdout, stderr, wait_thr = Open3.popen3(env, *command)
+        stdin.close
 
         @threads <<
-          Thread.new {
-            require "json"
-            stdout.each_line do |line|
-              result = line.split(env["RSPEC_FORMATTER_OUTPUT_ID"])
-
-              output = result.shift
-              STDOUT.print(output) unless output.empty?
-
-              message = result.shift
-              next unless message
-
-              message = JSON.parse(message)
-              message["process_id"] = process_id
-              @messages << message
+          Thread.new do
+            File.open(tmp_filename) do |fd|
+              fd.each_line do |line|
+                message = JSON.parse(line, symbolize_names: true)
+
+                message[:process_id] = process_id
+                @messages << message
+              end
             end
 
-            @messages << {"type" => "exit", "process_id" => process_id}
-          }
+            @messages << {type: "exit", process_id: process_id}
+          end
 
+        @threads << start_copy_thread(stdout, STDOUT)
         @threads << start_copy_thread(stderr, STDERR)
+
+        @threads << Thread.new {
+          unless wait_thr.value.success?
+            @messages << {type: "error"}
+          end
+        }
+
+        wait_thr
       end
     end
 
@@ -141,6 +198,7 @@ module TurboTests
         loop do
           msg = src.readpartial(4096)
         rescue EOFError
+          src.close
           break
         else
           dst.write(msg)
@@ -153,31 +211,47 @@ module TurboTests
 
       loop do
         message = @messages.pop
-        case message["type"]
+        case message[:type]
         when "example_passed"
-          example = FakeExample.from_obj(message["example"])
+          example = FakeExample.from_obj(message[:example])
           @reporter.example_passed(example)
+        when "group_started"
+          @reporter.group_started(message[:group].to_struct)
+        when "group_finished"
+          @reporter.group_finished
         when "example_pending"
-          example = FakeExample.from_obj(message["example"])
+          example = FakeExample.from_obj(message[:example])
           @reporter.example_pending(example)
+        when "load_summary"
+          message = message[:summary]
+          # NOTE: notifications order and content is not guaranteed hence the fetch
+          #       and count increment tracking to get the latest accumulated load time
+          @reporter.load_time = message[:load_time] if message.fetch(:count, 0) > @load_count
         when "example_failed"
-          example = FakeExample.from_obj(message["example"])
+          example = FakeExample.from_obj(message[:example])
           @reporter.example_failed(example)
           @failure_count += 1
           if fail_fast_met
             @threads.each(&:kill)
             break
           end
+        when "message"
+          @reporter.message(message[:message])
         when "seed"
         when "close"
+        when "error"
+          @reporter.error_outside_of_examples
+          @error = true
         when "exit"
           exited += 1
           if exited == @num_processes
             break
           end
         else
-          warn("Unhandled message in main process: #{message}")
+          STDERR.puts("Unhandled message in main process: #{message}")
         end
+
+        STDOUT.flush
       end
     rescue Interrupt
     end
@@ -186,14 +260,12 @@ module TurboTests
       !@fail_fast.nil? && @fail_fast >= @failure_count
     end
 
-    private
-
     def report_number_of_tests(groups)
       name = ParallelTests::RSpec::Runner.test_file_name
 
       num_processes = groups.size
       num_tests = groups.map(&:size).sum
-      tests_per_process = (num_processes == 0 ? 0 : num_tests / num_processes)
+      tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round
 
       puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process"
     end

data/lib/turbo_tests/version.rb

@@ -1,3 +1,3 @@
 module TurboTests
-  VERSION = "1.2.0"
+  VERSION = "1.2.5"
 end

data/lib/utils/hash_extension.rb

@@ -0,0 +1,9 @@
+module CoreExtensions
+  refine Hash do
+    def to_struct
+      OpenStruct.new(self.each_with_object({}) do |(key, val), acc|
+        acc[key] = val.is_a?(Hash) ? val.to_struct : val
+      end)
+    end
+  end
+end

data/turbo_tests.gemspec

@@ -14,12 +14,12 @@ Gem::Specification.new do |spec|
   spec.metadata["source_code_uri"] = "https://github.com/serpapi/turbo_tests"
   spec.metadata["changelog_uri"] = "https://github.com/serpapi/turbo_tests/releases"
 
-  spec.required_ruby_version = ">= 2.4.0"
+  spec.required_ruby_version = ">= 2.4"
 
-  spec.add_dependency "rspec", "~> 3.10.0"
+  spec.add_dependency "rspec", "~> 3.10"
   spec.add_dependency "parallel_tests", "~> 3.3"
 
-  spec.add_development_dependency "pry", "~> 0.13"
+  spec.add_development_dependency "pry", "~> 0.14"
 
   spec.add_runtime_dependency "bundler"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbo_tests
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.5
 platform: ruby
 authors:
 - Ilya Zub
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-23 00:00:00.000000000 Z
+date: 2021-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.10.0
+        version: '3.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.10.0
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: parallel_tests
   requirement: !ruby/object:Gem::Requirement
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -74,6 +74,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/snyk_ruby-analysis.yml"
 - ".github/workflows/tests.yml"
 - ".gitignore"
 - ".rspec"
@@ -85,12 +86,15 @@ files:
 - README.md
 - Rakefile
 - bin/turbo_tests
+- fixtures/rspec/errors_outside_of_examples_spec.rb
+- fixtures/rspec/pending_exceptions_spec.rb
 - lib/turbo_tests.rb
 - lib/turbo_tests/cli.rb
 - lib/turbo_tests/json_rows_formatter.rb
 - lib/turbo_tests/reporter.rb
 - lib/turbo_tests/runner.rb
 - lib/turbo_tests/version.rb
+- lib/utils/hash_extension.rb
 - turbo_tests.gemspec
 homepage: https://github.com/serpapi/turbo_tests
 licenses:
@@ -107,7 +111,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.4.0
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="