turbo_tests 1.0.0 → 1.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90e474a8156b2c560bca5ee96efb75dd9d124a97e7a088bcc8b92223f0dd7580
-  data.tar.gz: 73a47005dbc152492b6ef7e7f4d3c80cfdd958d755f998db453bc1f2cf7f48a8
+  metadata.gz: 748a8d1ce5e4bd659cbef9af59b3f896b2c5b1940ae252a1a9c000c8a0237f39
+  data.tar.gz: 30dfc4c96dd9d624114fba30f628b500396da5775db3298b8aefeaca1407eb9f
 SHA512:
-  metadata.gz: 9956bf53df8328cff65ef313397f358607ea359d23d086a07389c9fdc673169919220273cfcf77df267f72d45774524ded5318720fd23c59962fcb62d7eb0c68
-  data.tar.gz: dcacad7897a69b0701993256fa2185454639c91c71810938e9cab574d0d1e601dacb74fe46b68930ddeaf8deba7a40fbc1a95b95b1fff055cffc894a191bf9e8
+  metadata.gz: 568f6fec127a34381085a8465f1929a6324a12e010a4cdb78ba3956cbf9922840b199f2472520a37f0a013dc8e78d143b80748a029d9f15999aa49d974447a56
+  data.tar.gz: f655769722f00cbe2e6d3bc756666c3bb0cababb646466717952f11c99c15868ff144e6bc321e488e2a57d07bb959b39e86f7dac321c0c05d67f24e943168867

data/.github/workflows/snyk_ruby-analysis.yml

@@ -0,0 +1,33 @@
+# A sample workflow which checks out your Infrastructure as Code Configuration files,
+# such as Kubernetes, Helm & Terraform and scans them for any security issues.
+# The results are then uploaded to GitHub Security Code Scanning
+#
+# For more examples, including how to limit scans to only high-severity issues
+# and fail PR checks, see https://github.com/snyk/actions/
+
+name: Snyk Ruby
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '0 0 * * 0'
+
+jobs:
+  snyk:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run Snyk to check configuration files for security issues
+        # Snyk can be used to break the build when it detects security issues.
+        # In this case we want to upload the issues to GitHub Code Scanning
+        continue-on-error: true
+        uses: snyk/actions/ruby@master
+        env:
+          # In order to use the Snyk Action you will need to have a Snyk API token.
+          # More details in https://github.com/snyk/actions#getting-your-snyk-token
+          # or you can signup for free at https://snyk.io/login
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

data/.github/workflows/tests.yml

@@ -0,0 +1,25 @@
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby:
+          - 2.6
+          - 2.7
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@master
+        with:
+          ruby-version: ${{matrix.ruby}}
+          bundler-cache: true
+
+      - name: Run tests
+        timeout-minutes: 5
+        run: |
+          ${{matrix.env}} bundle exec turbo_tests -n4

data/.gitignore

@@ -9,3 +9,60 @@
 
 # rspec failure tracking
 .rspec_status
+
+
+# Created by https://www.toptal.com/developers/gitignore/api/macos,vim
+# Edit at https://www.toptal.com/developers/gitignore?templates=macos,vim
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/macos,vim
+

data/.rspec

@@ -1,3 +1,4 @@
 --format documentation
 --color
+--tty
 --require spec_helper

data/Gemfile

@@ -3,4 +3,4 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in turbo_tests.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -1,10 +1,10 @@
 PATH
   remote: .
   specs:
-    turbo_tests (0.1.0)
-      optparse
-      parallel_tests
-      rspec
+    turbo_tests (1.2.4)
+      bundler
+      parallel_tests (~> 3.3)
+      rspec (~> 3.10.0)
 
 GEM
   remote: https://rubygems.org/
@@ -12,34 +12,33 @@ GEM
     coderay (1.1.3)
     diff-lcs (1.4.4)
     method_source (1.0.0)
-    optparse (0.1.0)
-    parallel (1.19.2)
-    parallel_tests (3.3.0)
+    parallel (1.20.1)
+    parallel_tests (3.5.2)
       parallel
-    pry (0.13.1)
+    pry (0.14.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rake (12.3.3)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.3)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.3)
+    rake (13.0.3)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.4)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  pry
-  rake (~> 12.0)
+  pry (~> 0.14)
+  rake (~> 13.0)
   turbo_tests!
 
 BUNDLED WITH

data/README.md

@@ -1,3 +1,5 @@
+![Tests](https://github.com/serpapi/turbo_tests/workflows/Tests/badge.svg)
+
 # TurboTests
 
 Runner for [`grosser/parallel_tests`](https://github.com/grosser/parallel_tests) with incremental summarized output. Based on [Discourse](https://github.com/discourse/discourse/blob/6b9784cf8a18636bce281a7e4d18e65a0cbc6290/lib/turbo_tests.rb) and [RubyGems](https://github.com/rubygems/rubygems/tree/390335ceb351668cd433bd5bb9823dd021f82533/bundler/tool) work in this area.
@@ -11,23 +13,17 @@ This feature [doesn't fit vision of `parallel_tests` author](https://github.com/
 ```bash
 
 $ bundle exec rake parallel_tests:spec[^spec/search]
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
+.................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
 
 Finished in 1 minute 6.92 seconds (files took 6.95 seconds to load)
 2616 examples, 0 failures
 
-.........................................................................................................................................F.................................................
-.......................................................................................F...................................................................................................
-...........................................................................................................................................................................................
+.........................................................................................................................................F........................................................................................................................................F..............................................................................................................................................................................................................................................................................................
 
 Finished in 1 minute 35.05 seconds (files took 6.26 seconds to load)
 2158 examples, 2 failures
 
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
+.................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
 
 Finished in 1 minute 35.05 seconds (files took 6.26 seconds to load)
 2158 examples, 0 failures
@@ -36,16 +32,8 @@ Finished in 1 minute 35.05 seconds (files took 6.26 seconds to load)
 `turbo_tests` output looks like regular `rspec`:
 
 ```bash
-$ bundle exec turbo_spec
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-.........................................................................................................................................F.................................................
-.......................................................................................F...................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
-...........................................................................................................................................................................................
+$ bundle exec turbo_tests
+..........................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................F........................................................................................................................................F..............................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................
 
 Finished in 2 minute 25.15 seconds (files took 0 seconds to load)
 6873 examples, 2 failures
@@ -80,17 +68,20 @@ Show help:
 ```bash
 $ bundle exec turbo_tests -h
 Usage: turbo_tests [options]
+
+[optional] Only selected files & folders:
+  turbo_tests spec/bar spec/baz/xxx_spec.rb
+
+Options:
+    -n [PROCESSES]                   How many processes to use, default: available CPUs
     -r, --require PATH               Require a file.
-    -f, --format FORMATTER           Choose a formatter.
+    -f, --format FORMATTER           Choose a formatter. Available formatters: progress (p), documentation (d). Default: progress
     -t, --tag TAG                    Run examples with the specified tag.
     -o, --out FILE                   Write output to a file instead of $stdout
     -v, --verbose                    More output
         --fail-fast=[N]
 ```
 
-## Roadmap
-- [ ] Capture time to load files #1
-
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -1,3 +1,4 @@
+require "bundler/setup"
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 

data/bin/turbo_tests

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+# Enable local usage from cloned repo
+root = File.expand_path("../..", __FILE__)
+$LOAD_PATH << "#{root}/lib" if File.exist?("#{root}/Gemfile")
+
+require "turbo_tests"
+
+TurboTests::CLI.new(ARGV).run

data/fixtures/rspec/errors_outside_of_examples_spec.rb

@@ -0,0 +1,5 @@
+RSpec.describe "Fixture of spec file with errors outside of examples" do
+  it("passes") { expect(2 * 2).to eql(4) }
+
+  1 / 0
+end

data/lib/turbo_tests.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "turbo_tests/version"
-require "bundler/setup"
-
 require "open3"
 require "fileutils"
 require "json"
@@ -16,6 +13,9 @@ require "turbo_tests/runner"
 require "turbo_tests/json_rows_formatter"
 
 module TurboTests
+  autoload :CLI, "turbo_tests/cli"
+  autoload :VERSION, "turbo_tests/version"
+
   FakeException = Struct.new(:backtrace, :message, :cause)
   class FakeException
     def self.from_obj(obj)
@@ -23,14 +23,14 @@ module TurboTests
         klass =
           Class.new(FakeException) {
             define_singleton_method(:name) do
-              obj["class_name"]
+              obj[:class_name]
             end
           }
 
         klass.new(
-          obj["backtrace"],
-          obj["message"],
-          FakeException.from_obj(obj["cause"])
+          obj[:backtrace],
+          obj[:message],
+          FakeException.from_obj(obj[:cause])
         )
       end
     end
@@ -40,11 +40,11 @@ module TurboTests
   class FakeExecutionResult
     def self.from_obj(obj)
       new(
-        obj["example_skipped?"],
-        obj["pending_message"],
-        obj["status"].to_sym,
-        obj["pending_fixed?"],
-        FakeException.from_obj(obj["exception"])
+        obj[:example_skipped?],
+        obj[:pending_message],
+        obj[:status].to_sym,
+        obj[:pending_fixed?],
+        FakeException.from_obj(obj[:exception])
       )
     end
   end
@@ -52,24 +52,24 @@ module TurboTests
   FakeExample = Struct.new(:execution_result, :location, :description, :full_description, :metadata, :location_rerun_argument)
   class FakeExample
     def self.from_obj(obj)
-      metadata = obj["metadata"]
+      metadata = obj[:metadata]
 
-      metadata["shared_group_inclusion_backtrace"].map! do |frame|
+      metadata[:shared_group_inclusion_backtrace].map! do |frame|
         RSpec::Core::SharedExampleGroupInclusionStackFrame.new(
-          frame["shared_group_name"],
-          frame["inclusion_location"]
+          frame[:shared_group_name],
+          frame[:inclusion_location]
         )
       end
 
-      metadata[:shared_group_inclusion_backtrace] = metadata.delete("shared_group_inclusion_backtrace")
+      metadata[:shared_group_inclusion_backtrace] = metadata.delete(:shared_group_inclusion_backtrace)
 
       new(
-        FakeExecutionResult.from_obj(obj["execution_result"]),
-        obj["location"],
-        obj["description"],
-        obj["full_description"],
+        FakeExecutionResult.from_obj(obj[:execution_result]),
+        obj[:location],
+        obj[:description],
+        obj[:full_description],
         metadata,
-        obj["location_rerun_argument"]
+        obj[:location_rerun_argument]
       )
     end
 

data/lib/turbo_tests/cli.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "turbo_tests"
 require "optparse"
 
 module TurboTests
@@ -13,15 +12,31 @@ module TurboTests
       requires = []
       formatters = []
       tags = []
+      count = nil
       verbose = false
       fail_fast = nil
 
       OptionParser.new { |opts|
+        opts.banner = <<~BANNER
+          Run all tests in parallel, giving each process ENV['TEST_ENV_NUMBER'] ('1', '2', '3', ...).
+
+          Uses parallel_tests under the hood, but reports test results incrementally. Based on Discourse and RubyGems work in this area.
+
+          Usage: turbo_tests [options]
+
+          [optional] Only selected files & folders:
+            turbo_tests spec/bar spec/baz/xxx_spec.rb
+
+          Options:
+        BANNER
+
+        opts.on("-n [PROCESSES]", Integer, "How many processes to use, default: available CPUs") { |n| count = n }
+
         opts.on("-r", "--require PATH", "Require a file.") do |filename|
           requires << filename
         end
 
-        opts.on("-f", "--format FORMATTER", "Choose a formatter.") do |name|
+        opts.on("-f", "--format FORMATTER", "Choose a formatter. Available formatters: progress (p), documentation (d). Default: progress") do |name|
           formatters << {
             name: name,
             outputs: []
@@ -71,13 +86,20 @@ module TurboTests
         end
       end
 
-      exit TurboTests::Runner.run(
+      success = TurboTests::Runner.run(
         formatters: formatters,
         tags: tags,
         files: @argv.empty? ? ["spec"] : @argv,
         verbose: verbose,
-        fail_fast: fail_fast
+        fail_fast: fail_fast,
+        count: count,
       )
+
+      if success
+        exit 0
+      else
+        exit 1
+      end
     end
   end
 end

data/lib/turbo_tests/json_rows_formatter.rb

@@ -22,10 +22,15 @@ module TurboTests
   class JsonRowsFormatter
     RSpec::Core::Formatters.register(
       self,
+      :start,
       :close,
       :example_failed,
       :example_passed,
       :example_pending,
+      :example_group_started,
+      :example_group_finished,
+      :example_pending,
+      :message,
       :seed
     )
 
@@ -35,37 +40,65 @@ module TurboTests
       @output = output
     end
 
+    def start(notification)
+      output_row(
+        type: :load_summary,
+        summary: load_summary_to_json(notification)
+      )
+    end
+
+    def example_group_started(notification)
+      output_row(
+        type: :group_started,
+        group: group_to_json(notification)
+      )
+    end
+
+    def example_group_finished(notification)
+      output_row(
+        type: :group_finished,
+        group: group_to_json(notification)
+      )
+    end
+
     def example_passed(notification)
       output_row(
-        "type" => :example_passed,
-        "example" => example_to_json(notification.example)
+        type: :example_passed,
+        example: example_to_json(notification.example)
       )
     end
 
     def example_pending(notification)
       output_row(
-        "type" => :example_pending,
-        "example" => example_to_json(notification.example)
+        type: :example_pending,
+        example: example_to_json(notification.example)
       )
     end
 
     def example_failed(notification)
       output_row(
-        "type" => :example_failed,
-        "example" => example_to_json(notification.example)
+        type: :example_failed,
+        example: example_to_json(notification.example)
       )
     end
 
     def seed(notification)
       output_row(
-        "type" => :seed,
-        "seed" => notification.seed
+        type: :seed,
+        seed: notification.seed
       )
     end
 
     def close(notification)
       output_row(
-        "type" => :close
+        type: :close
+      )
+    end
+
+    def message(notification)
+      output_row(
+        type: :message,
+        message: notification.message
       )
     end
 
@@ -74,47 +107,65 @@ module TurboTests
     def exception_to_json(exception)
       if exception
         {
-          "class_name" => exception.class.name.to_s,
-          "backtrace" => exception.backtrace,
-          "message" => exception.message,
-          "cause" => exception_to_json(exception.cause)
+          class_name: exception.class.name.to_s,
+          backtrace: exception.backtrace,
+          message: exception.message,
+          cause: exception_to_json(exception.cause)
         }
       end
     end
 
     def execution_result_to_json(result)
       {
-        "example_skipped?" => result.example_skipped?,
-        "pending_message" => result.pending_message,
-        "status" => result.status,
-        "pending_fixed?" => result.pending_fixed?,
-        "exception" => exception_to_json(result.exception)
+        example_skipped?: result.example_skipped?,
+        pending_message: result.pending_message,
+        status: result.status,
+        pending_fixed?: result.pending_fixed?,
+        exception: exception_to_json(result.exception)
       }
     end
 
     def stack_frame_to_json(frame)
       {
-        "shared_group_name" => frame.shared_group_name,
-        "inclusion_location" => frame.inclusion_location
+        shared_group_name: frame.shared_group_name,
+        inclusion_location: frame.inclusion_location
       }
     end
 
     def example_to_json(example)
       {
-        "execution_result" => execution_result_to_json(example.execution_result),
-        "location" => example.location,
-        "description" => example.description,
-        "full_description" => example.full_description,
-        "metadata" => {
-          "shared_group_inclusion_backtrace" =>
-            example.metadata[:shared_group_inclusion_backtrace].map { |frame| stack_frame_to_json(frame) }
+        execution_result: execution_result_to_json(example.execution_result),
+        location: example.location,
+        description: example.description,
+        full_description: example.full_description,
+        metadata: {
+          shared_group_inclusion_backtrace:
+            example
+              .metadata[:shared_group_inclusion_backtrace]
+              .map { |frame| stack_frame_to_json(frame) }
         },
-        "location_rerun_argument" => example.location_rerun_argument
+        location_rerun_argument: example.location_rerun_argument
+      }
+    end
+
+    def load_summary_to_json(notification)
+      {
+        count: notification.count,
+        load_time: notification.load_time,
+      }
+    end
+
+    def group_to_json(notification)
+      {
+        group: {
+          description: notification.group.description
+        }
       }
     end
 
     def output_row(obj)
-      output.puts ENV["RSPEC_FORMATTER_OUTPUT_ID"] + obj.to_json
+      output.puts(obj.to_json)
+      output.flush
     end
   end
 end

data/lib/turbo_tests/reporter.rb

@@ -2,6 +2,8 @@
 
 module TurboTests
   class Reporter
+    attr_writer :load_time
+
     def self.from_config(formatter_config, start_time)
       reporter = new(start_time)
 
@@ -26,7 +28,10 @@ module TurboTests
       @pending_examples = []
       @failed_examples = []
       @all_examples = []
+      @messages = []
       @start_time = start_time
+      @load_time = 0
+      @errors_outside_of_examples_count = 0
     end
 
     def add(name, outputs)
@@ -45,6 +50,14 @@ module TurboTests
       end
     end
 
+    def group_started(notification)
+      delegate_to_formatters(:example_group_started, notification)
+    end
+
+    def group_finished
+      delegate_to_formatters(:example_group_finished, nil)
+    end
+
     def example_passed(example)
       delegate_to_formatters(:example_passed, example.notification)
 
@@ -65,8 +78,18 @@ module TurboTests
       @failed_examples << example
     end
 
+    def message(message)
+      delegate_to_formatters(:message, RSpec::Core::Notifications::MessageNotification.new(message))
+      @messages << message
+    end
+
+    def error_outside_of_examples
+      @errors_outside_of_examples_count += 1
+    end
+
     def finish
-      end_time = Time.now
+      # SEE: https://bit.ly/2NP87Cz
+      end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
       delegate_to_formatters(:start_dump,
         RSpec::Core::Notifications::NullNotification)
@@ -84,8 +107,8 @@ module TurboTests
           @all_examples,
           @failed_examples,
           @pending_examples,
-          0,
-          0
+          @load_time,
+          @errors_outside_of_examples_count
         ))
       delegate_to_formatters(:close,
         RSpec::Core::Notifications::NullNotification)

data/lib/turbo_tests/runner.rb

@@ -1,18 +1,28 @@
 # frozen_string_literal: true
 
 require "json"
-
 require "parallel_tests/rspec/runner"
 
+require_relative "../utils/hash_extension"
+
 module TurboTests
   class Runner
+    using CoreExtensions
+
     def self.run(opts = {})
       files = opts[:files]
       formatters = opts[:formatters]
       tags = opts[:tags]
-      start_time = opts.fetch(:start_time) { Time.now }
+
+      # SEE: https://bit.ly/2NP87Cz
+      start_time = opts.fetch(:start_time) { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
       verbose = opts.fetch(:verbose, false)
       fail_fast = opts.fetch(:fail_fast, nil)
+      count = opts.fetch(:count, nil)
+
+      if verbose
+        STDERR.puts "VERBOSE"
+      end
 
       reporter = Reporter.from_config(formatters, start_time)
 
@@ -21,7 +31,8 @@ module TurboTests
         files: files,
         tags: tags,
         verbose: verbose,
-        fail_fast: fail_fast
+        fail_fast: fail_fast,
+        count: count
       ).run
     end
 
@@ -31,25 +42,49 @@ module TurboTests
       @tags = opts[:tags]
       @verbose = opts[:verbose]
       @fail_fast = opts[:fail_fast]
+      @count = opts[:count]
+      @load_time = 0
+      @load_count = 0
       @failure_count = 0
-      @runtime_log = "tmp/parallel_runtime_rspec.log"
 
       @messages = Queue.new
       @threads = []
+      @error = false
     end
 
     def run
-      @num_processes = ParallelTests.determine_number_of_processes(nil)
+      @num_processes = [
+        ParallelTests.determine_number_of_processes(@count),
+        ParallelTests::RSpec::Runner.tests_with_size(@files, {}).size
+      ].min
+
+      use_runtime_info = @files == ["spec"]
+
+      group_opts = {}
+
+      if use_runtime_info
+        group_opts[:runtime_log] = "tmp/turbo_rspec_runtime.log"
+      else
+        group_opts[:group_by] = :filesize
+      end
 
       tests_in_groups =
         ParallelTests::RSpec::Runner.tests_in_groups(
           @files,
           @num_processes,
-          runtime_log: @runtime_log
+          **group_opts
         )
 
-      tests_in_groups.each_with_index do |tests, process_id|
-        start_regular_subprocess(tests, process_id + 1)
+      setup_tmp_dir
+
+      subprocess_opts = {
+        record_runtime: use_runtime_info
+      }
+
+      report_number_of_tests(tests_in_groups)
+
+      wait_threads = tests_in_groups.map.with_index do |tests, process_id|
+        start_regular_subprocess(tests, process_id + 1, **subprocess_opts)
       end
 
       handle_messages
@@ -58,37 +93,64 @@ module TurboTests
 
       @threads.each(&:join)
 
-      @reporter.failed_examples.empty?
+      @reporter.failed_examples.empty? && wait_threads.map(&:value).all?(&:success?)
     end
 
-    protected
+    private
+
+    def setup_tmp_dir
+      begin
+        FileUtils.rm_r("tmp/test-pipes")
+      rescue Errno::ENOENT
+      end
+
+      FileUtils.mkdir_p("tmp/test-pipes/")
+    end
 
-    def start_regular_subprocess(tests, process_id)
+    def start_regular_subprocess(tests, process_id, **opts)
       start_subprocess(
         {"TEST_ENV_NUMBER" => process_id.to_s},
         @tags.map { |tag| "--tag=#{tag}" },
         tests,
-        process_id
+        process_id,
+        **opts
       )
     end
 
-    def start_subprocess(env, extra_args, tests, process_id)
+    def start_subprocess(env, extra_args, tests, process_id, record_runtime:)
       if tests.empty?
         @messages << {
-          "type" => "exit",
-          "process_id" => process_id
+          type: "exit",
+          process_id: process_id
         }
       else
-        require "securerandom"
-        env["RSPEC_FORMATTER_OUTPUT_ID"] = SecureRandom.uuid
-        env["RUBYOPT"] = "-I#{File.expand_path("..", __dir__)}"
+        tmp_filename = "tmp/test-pipes/subprocess-#{process_id}"
+
+        begin
+          File.mkfifo(tmp_filename)
+        rescue Errno::EEXIST
+        end
+
+        env["RUBYOPT"] = ["-I#{File.expand_path("..", __dir__)}", ENV["RUBYOPT"]].compact.join(" ")
+        env["RSPEC_SILENCE_FILTER_ANNOUNCEMENTS"] = "1"
+
+        record_runtime_options =
+          if record_runtime
+            [
+              "--format", "ParallelTests::RSpec::RuntimeLogger",
+              "--out", "tmp/turbo_rspec_runtime.log",
+            ]
+          else
+            []
+          end
 
         command = [
-          "bundle", "exec", "rspec",
+          ENV["BUNDLE_BIN_PATH"], "exec", "rspec",
           *extra_args,
-          "--format", "ParallelTests::RSpec::RuntimeLogger",
-          "--out", @runtime_log,
+          "--seed", rand(0xFFFF).to_s,
           "--format", "TurboTests::JsonRowsFormatter",
+          "--out", tmp_filename,
+          *record_runtime_options,
           *tests
         ]
 
@@ -101,29 +163,33 @@ module TurboTests
           STDERR.puts "Process #{process_id}: #{command_str}"
         end
 
-        _stdin, stdout, stderr, _wait_thr = Open3.popen3(env, *command)
+        stdin, stdout, stderr, wait_thr = Open3.popen3(env, *command)
+        stdin.close
 
         @threads <<
-          Thread.new {
-            require "json"
-            stdout.each_line do |line|
-              result = line.split(env["RSPEC_FORMATTER_OUTPUT_ID"])
+          Thread.new do
+            File.open(tmp_filename) do |fd|
+              fd.each_line do |line|
+                message = JSON.parse(line, symbolize_names: true)
 
-              output = result.shift
-              STDOUT.print(output) unless output.empty?
-
-              message = result.shift
-              next unless message
-
-              message = JSON.parse(message)
-              message["process_id"] = process_id
-              @messages << message
+                message[:process_id] = process_id
+                @messages << message
+              end
             end
 
-            @messages << {"type" => "exit", "process_id" => process_id}
-          }
+            @messages << {type: "exit", process_id: process_id}
+          end
 
+        @threads << start_copy_thread(stdout, STDOUT)
         @threads << start_copy_thread(stderr, STDERR)
+
+        @threads << Thread.new {
+          unless wait_thr.value.success?
+            @messages << {type: "error"}
+          end
+        }
+
+        wait_thr
       end
     end
 
@@ -132,6 +198,7 @@ module TurboTests
         loop do
           msg = src.readpartial(4096)
         rescue EOFError
+          src.close
           break
         else
           dst.write(msg)
@@ -144,31 +211,47 @@ module TurboTests
 
       loop do
         message = @messages.pop
-        case message["type"]
+        case message[:type]
         when "example_passed"
-          example = FakeExample.from_obj(message["example"])
+          example = FakeExample.from_obj(message[:example])
           @reporter.example_passed(example)
+        when "group_started"
+          @reporter.group_started(message[:group].to_struct)
+        when "group_finished"
+          @reporter.group_finished
         when "example_pending"
-          example = FakeExample.from_obj(message["example"])
+          example = FakeExample.from_obj(message[:example])
           @reporter.example_pending(example)
+        when "load_summary"
+          message = message[:summary]
+          # NOTE: notifications order and content is not guaranteed hence the fetch
+          #       and count increment tracking to get the latest accumulated load time
+          @reporter.load_time = message[:load_time] if message.fetch(:count, 0) > @load_count
         when "example_failed"
-          example = FakeExample.from_obj(message["example"])
+          example = FakeExample.from_obj(message[:example])
           @reporter.example_failed(example)
           @failure_count += 1
           if fail_fast_met
             @threads.each(&:kill)
             break
           end
+        when "message"
+          @reporter.message(message[:message])
         when "seed"
         when "close"
+        when "error"
+          @reporter.error_outside_of_examples
+          @error = true
         when "exit"
           exited += 1
           if exited == @num_processes
             break
           end
         else
-          warn("Unhandled message in main process: #{message}")
+          STDERR.puts("Unhandled message in main process: #{message}")
         end
+
+        STDOUT.flush
       end
     rescue Interrupt
     end
@@ -176,5 +259,15 @@ module TurboTests
     def fail_fast_met
       !@fail_fast.nil? && @fail_fast >= @failure_count
     end
+
+    def report_number_of_tests(groups)
+      name = ParallelTests::RSpec::Runner.test_file_name
+
+      num_processes = groups.size
+      num_tests = groups.map(&:size).sum
+      tests_per_process = (num_processes == 0 ? 0 : num_tests.to_f / num_processes).round
+
+      puts "#{num_processes} processes for #{num_tests} #{name}s, ~ #{tests_per_process} #{name}s per process"
+    end
   end
 end

data/lib/turbo_tests/version.rb

@@ -1,3 +1,3 @@
 module TurboTests
-  VERSION = "1.0.0"
+  VERSION = "1.2.4"
 end

data/lib/utils/hash_extension.rb

@@ -0,0 +1,9 @@
+module CoreExtensions
+  refine Hash do
+    def to_struct
+      OpenStruct.new(self.each_with_object({}) do |(key, val), acc|
+        acc[key] = val.is_a?(Hash) ? val.to_struct : val
+      end)
+    end
+  end
+end

data/turbo_tests.gemspec

@@ -4,22 +4,24 @@ Gem::Specification.new do |spec|
   spec.name = "turbo_tests"
   spec.version = TurboTests::VERSION
   spec.authors = ["Ilya Zub"]
-  spec.email = ["ilya@serpapi.com"]
+  spec.email = ["zaoooza92@gmail.com"]
 
   spec.summary = "Runner for grosser/parallel_tests with incremental summarized output. Based on Discourse and Rubygems work in this area."
   spec.homepage = "https://github.com/serpapi/turbo_tests"
   spec.license = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/serpapi/turbo_tests"
   spec.metadata["changelog_uri"] = "https://github.com/serpapi/turbo_tests/releases"
 
-  spec.add_dependency "rspec"
-  spec.add_dependency "parallel_tests"
-  spec.add_dependency "optparse"
+  spec.required_ruby_version = ">= 2.4.0"
 
-  spec.add_development_dependency "pry"
+  spec.add_dependency "rspec", "~> 3.10.0"
+  spec.add_dependency "parallel_tests", "~> 3.3"
+
+  spec.add_development_dependency "pry", "~> 0.14"
+
+  spec.add_runtime_dependency "bundler"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -27,7 +29,5 @@ Gem::Specification.new do |spec|
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
 
-  spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.executables = ["turbo_tests"]
 end

metadata

@@ -1,65 +1,65 @@
 --- !ruby/object:Gem::Specification
 name: turbo_tests
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.4
 platform: ruby
 authors:
 - Ilya Zub
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2020-10-30 00:00:00.000000000 Z
+date: 2021-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.10.0
 - !ruby/object:Gem::Dependency
   name: parallel_tests
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.3'
 - !ruby/object:Gem::Dependency
-  name: optparse
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '0.14'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.14'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -68,12 +68,14 @@ dependencies:
         version: '0'
 description: 
 email:
-- ilya@serpapi.com
+- zaoooza92@gmail.com
 executables:
 - turbo_tests
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/snyk_ruby-analysis.yml"
+- ".github/workflows/tests.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -83,15 +85,15 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
-- exe/turbo_tests
+- bin/turbo_tests
+- fixtures/rspec/errors_outside_of_examples_spec.rb
 - lib/turbo_tests.rb
 - lib/turbo_tests/cli.rb
 - lib/turbo_tests/json_rows_formatter.rb
 - lib/turbo_tests/reporter.rb
 - lib/turbo_tests/runner.rb
 - lib/turbo_tests/version.rb
+- lib/utils/hash_extension.rb
 - turbo_tests.gemspec
 homepage: https://github.com/serpapi/turbo_tests
 licenses:
@@ -108,7 +110,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/bin/console

@@ -1,7 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "turbo_tests"
-
-require "pry"
-Pry.start

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/exe/turbo_tests

@@ -1,7 +0,0 @@
-#!/usr/bin/env ruby
-
-# frozen_string_literal: true
-
-require "turbo_tests/cli"
-
-TurboTests::CLI.new(ARGV).run