rspec-sharder 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6d525960fabd4fd6d3c83ce963601ca0552bbcb15e11c212cdf872de2cd71a36
+  data.tar.gz: 5c7591851f241b1ab2b017ad81f3488bd5ac6dea21c1978df4e2cb963bd17bf2
+SHA512:
+  metadata.gz: 1c65df78f7e5d973308808db3c48309c14d3e0abd1d7b5ccc37d1faa1d2f82b1ff98acba5a38f39947f47927cffaedf4dfe3f4b744b4cceb11959df2762b5bf7
+  data.tar.gz: db806df9aaa7f00cf7f62a2f583908a1d98864d2dd34b93a61a477c35716fa55f3f42c2cf9f22b9be49474e0d164f01776f2d1d8fbe4ce7ecbc05c9ebb3ab9dd

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+ruby '3.0.0'
+
+source "https://rubygems.org"
+
+gemspec
+
+group :test do
+  gem 'rspec'
+end

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    rspec-sharder (0.0.1)
+      rspec-core
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.4.4)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  rspec
+  rspec-sharder!
+
+RUBY VERSION
+   ruby 3.0.0p0
+
+BUNDLED WITH
+   2.2.21

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Nick Dower
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,26 @@
+# RSpec Sharder
+
+```
+Groups specs into shards, ensuring that each shard has a similar size, and runs
+the specified shard.
+
+Shard size is determined by summing the saved durations for each spec file in
+the shard. Durations are saved in .spec_durations. If a spec file is not found
+in .spec_durations, the duration is estimated based on the number of examples in
+the spec file.
+
+.spec_durations is generate/updated after a successful run when --persist is
+specified, but only for the shard which was actually executed. To generate
+durations for all shards simultaneously, run with the default options of 1 total
+shards and --persist:
+
+bundle exec rspec-sharder --persist -- [<rspec-args...>]
+
+Usage: bundle exec rspec-sharder [--total-shards <num> [--shard <num>]] [--persist] -- [<rspec-args...>]
+
+Options:
+    -h, --help                       Print this message.
+    -t, --total-shards <num>         The total number of shards. Defaults to 1.
+    -s, --shard <num>                The shard to run. Defaults to 1.
+    -p, --persist                    Save durations to .spec_durations.
+```

data/bin/rspec-sharder

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'rspec-sharder'
+
+def fail(message)
+  warn message
+  puts
+  puts @parser.help
+  exit 1
+end
+
+@total_shards = 1
+@shard_num = 1
+@persist = false
+
+@parser = OptionParser.new do |opts|
+  opts.banner = <<~EOF
+    Groups specs into shards, ensuring that each shard has a similar size, and runs
+    the specified shard.
+
+    Shard size is determined by summing the saved durations for each spec file in
+    the shard. Durations are saved in .spec_durations. If a spec file is not found
+    in .spec_durations, the duration is estimated based on the number of examples in
+    the spec file.
+
+    .spec_durations is generate/updated after a successful run when --persist is
+    specified, but only for the shard which was actually executed. To generate
+    durations for all shards simultaneously, run with the default options of 1 total
+    shards and --persist:
+
+    bundle exec rspec-sharder --persist -- [<rspec-args...>]
+
+    Usage: bundle exec rspec-sharder [--total-shards <num> [--shard <num>]] [--persist] -- [<rspec-args...>]
+
+    Options:
+  EOF
+
+  opts.on('-h', '--help', "Print this message.") do
+    puts opts
+    exit
+  end
+
+  opts.on('-t', '--total-shards <num>', 'The total number of shards. Defaults to 1.') do |total_shards|
+    begin
+      @total_shards = Integer(total_shards)
+    rescue ArgumentError
+      fail('fatal: invalid value for --total-shards')
+    end
+  end
+
+  opts.on('-s', '--shard <num>', 'The shard to run. Defaults to 1.') do |shard|
+    begin
+      @shard = Integer(shard)
+    rescue ArgumentError
+      fail('fatal: invalid value for --shard')
+    end
+  end
+
+  opts.on('-p', '--persist', 'Save durations to .spec_durations.') do
+    @persist = true
+  end
+end
+
+begin
+  @parser.parse!
+rescue StandardError => e
+  fail("fatal: #{e.message}")
+end
+
+fail('fatal: invalid value for --total-shards') unless @total_shards > 0
+fail('fatal: invalid value for --shard') unless @shard > 0
+fail('fatal: --shard may not be greater than --total-shards') unless @shard <= @total_shards
+
+RSpec::Sharder.run(total_shards: @total_shards, shard_num: @shard, persist: @persist, rspec_args: ARGV)

data/lib/rspec-sharder/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RSpec
+  module Sharder
+    VERSION = "0.0.1"
+  end
+end

data/lib/rspec-sharder.rb

@@ -0,0 +1,261 @@
+require 'rspec/core'
+
+module RSpec
+  module Sharder
+
+    class ShardError < StandardError; end
+
+    def self.run(total_shards:, shard_num:, persist:, rspec_args:)
+      raise "fatal: invalid total shards: #{total_shards}" unless total_shards.is_a?(Integer) && total_shards > 0
+      raise "fatal: invalid shard number: #{shard_num}" unless shard_num.is_a?(Integer) && shard_num > 0 && shard_num <= total_shards
+
+      begin
+        ::RSpec::Core::ConfigurationOptions.new(rspec_args).configure(::RSpec.configuration)
+
+        return if ::RSpec.world.wants_to_quit
+
+        ::RSpec.configuration.load_spec_files
+      ensure
+        ::RSpec.world.announce_filters
+      end
+
+      return ::RSpec.configuration.reporter.exit_early(::RSpec.configuration.failure_exit_code) if ::RSpec.world.wants_to_quit
+
+      all_durations = load_recorded_durations
+
+      begin
+        shards = build_shards(total_shards, shard_num, all_durations)
+      rescue ShardError => e
+        ::RSpec.configuration.error_stream.puts e.message
+        exit ::RSpec.configuration.failure_exit_cod
+      end
+
+      print_shards(shards)
+
+      expected_total_duration = shards[shard_num - 1][:duration]
+
+      shard_file_paths = shards[shard_num - 1][:file_paths]
+      example_groups = ::RSpec.world.ordered_example_groups.select do |example_group|
+        shard_file_paths.include?(example_group.metadata[:file_path])
+      end
+      example_count = ::RSpec.world.example_count(example_groups)
+
+      new_durations = { }
+
+      actual_total_duration = 0
+      exit_code = ::RSpec.configuration.reporter.report(example_count) do |reporter|
+        ::RSpec.configuration.with_suite_hooks do
+          if example_count == 0 && ::RSpec.configuration.fail_if_no_examples
+            return ::RSpec.configuration.failure_exit_code
+          end
+
+          group_results = example_groups.map do |example_group|
+            start_time = current_time_millis
+            result = example_group.run(reporter)
+            end_time = current_time_millis
+
+            file_path = example_group.metadata[:file_path]
+            duration = (end_time - start_time).to_i
+            actual_total_duration += duration
+            new_durations[file_path] ||= 0
+            new_durations[file_path] += duration
+
+            result
+          end
+
+          success = group_results.all?
+          exit_code = success ? 0 : 1
+          if ::RSpec.world.non_example_failure
+            success = false
+            exit_code = ::RSpec.configuration.failure_exit_code
+          end
+          exit_code
+        end
+      end
+
+      # Write results to .examples file.
+      unless ::RSpec.configuration.dry_run
+        persist_example_statuses(shard_file_paths)
+      end
+
+      if ::RSpec.configuration.dry_run
+        if persist
+          ::RSpec.configuration.output_stream.puts <<~EOF
+            
+            Dry run. Not saving to .spec_durations.
+          EOF
+        end
+      else
+        if exit_code == 0
+          # Print recorded durations and summary.
+          ::RSpec.configuration.output_stream.puts <<~EOF
+            
+            Durations:
+          EOF
+
+          new_durations.sort_by { |file_path, duration| file_path }.each do |file_path, duration|
+            ::RSpec.configuration.output_stream.puts "#{file_path},#{duration}"
+          end
+
+          ::RSpec.configuration.output_stream.puts <<~EOF
+            
+            Expected total duration: #{pretty_duration(expected_total_duration)}
+            Actual total duration:   #{pretty_duration(actual_total_duration)}
+            Diff:                    #{pretty_duration((actual_total_duration - expected_total_duration).abs)}
+          EOF
+
+          if persist
+            # Write all durations with updates to .spec_durations.
+            ::RSpec.configuration.output_stream.puts <<~EOF
+
+              Saving to .spec_durations.
+            EOF
+
+            new_durations.each do |file_path, duration|
+              all_durations[file_path] = duration
+            end
+
+            persist_durations(all_durations)
+          end
+        elsif persist
+          ::RSpec.configuration.output_stream.puts <<~EOF
+            
+            RSpec failed. Not saving to .spec_durations.
+          EOF
+        end
+      end
+
+      exit exit_code
+    end
+
+    private
+
+    def self.load_recorded_durations
+      durations = { }
+
+      if File.exist?('.spec_durations')
+        File.readlines('.spec_durations').each_with_index do |line, index|
+          line = line.strip
+
+          if !line.start_with?('#') && !line.empty?
+            parts = line.split(',')
+
+            unless parts.length == 2
+              raise ShardError.new("fatal: invalid .spec_durations at line #{index + 1}")
+            end
+
+            file_path = parts[0].strip
+
+            if file_path.empty?
+              raise ShardError.new("fatal: invalid file path in .spec_durations at line #{index + 1}")
+            end
+
+            unless File.exist?(file_path)
+              raise ShardError.new("fatal: file in .spec_durations not found at line #{index + 1}")
+            end
+
+            begin
+              duration = Integer(parts[1])
+            rescue ArgumentError => e
+              raise ShardError.new("fatal: invalid .spec_durations at line #{index + 1}")
+            end
+
+            durations[file_path] = duration
+          end
+        end.compact
+      end
+
+      durations
+    end
+
+    def self.build_shards(total_shards, shard_num, durations)
+      files = { }
+
+      ::RSpec.world.ordered_example_groups.each do |example_group|
+        file_path = example_group.metadata[:file_path]
+        files[file_path] ||= 0
+        if durations[file_path]
+          files[file_path] = durations[file_path]
+        else
+          ::RSpec.configuration.error_stream.puts "warning: recorded duration not found for #{file_path}"
+
+          # Assume 1000 milliseconds per example.
+          files[file_path] += ::RSpec.world.example_count([example_group]) * 1000
+        end
+      end
+
+      shards = (1..total_shards).map { { duration: 0, file_paths: [] } }
+
+      # First sort by duration to ensure large files are distributed evenly.
+      # Next, sort by path to ensure shards are generated deterministically.
+      # Note that files is a map, sorting it turns it into an array of arrays.
+      files = files.sort_by { |file_path, duration| [duration, file_path] }.reverse
+      files.each do |file_path, duration|
+        shards.sort_by! { |shard| shard[:duration] }
+        shards[0][:file_paths] << file_path
+        shards[0][:duration] += duration
+      end
+
+      shards.each { |shard| shard[:file_paths].sort! }
+
+      shards
+    end
+
+    def self.persist_example_statuses(file_paths)
+      return unless (path = ::RSpec.configuration.example_status_persistence_file_path)
+
+      examples = ::RSpec.world.all_examples.select do |example|
+        file_paths.include?(example.metadata[:file_path])
+      end
+      ::RSpec::Core::ExampleStatusPersister.persist(examples, path)
+    rescue SystemCallError => e
+      ::RSpec.configuration.error_stream.puts "warning: failed to write results to #{path}"
+    end
+
+    def self.pretty_duration(duration_millis)
+      duration_seconds = (duration_millis / 1000.0).round
+      minutes = duration_seconds / 60
+      seconds = duration_seconds % 60
+
+      minutes_str = "#{minutes} minute#{minutes == 1 ? '' : 's'}"
+      seconds_str = "#{seconds} second#{seconds == 1 ? '' : 's'}"
+
+      if minutes == 0
+        seconds_str
+      else
+        "#{minutes_str}, #{seconds_str}"
+      end
+    end
+
+    def self.print_shards(shards)
+      ::RSpec.configuration.output_stream.puts
+      shards.each_with_index do |shard, i|
+        ::RSpec.configuration.output_stream.puts(
+          "Shard #{i + 1} (Files: #{shard[:file_paths].size}, Duration: #{pretty_duration(shard[:duration])}):"
+        )
+        shard[:file_paths].each do |file_path|
+          ::RSpec.configuration.output_stream.puts file_path
+        end
+        ::RSpec.configuration.output_stream.puts
+      end
+    end
+
+    def self.persist_durations(durations)
+      File.open(".spec_durations", "w+") do |file|
+        file.puts <<~EOF
+          # This file was created by rspec-sharder on #{Time.now.to_s}.
+          # It is used to shard specs evenly. If test shards are uneven, run:
+          # 
+          #   bundle exec rspec-sharder --help
+        EOF
+        durations.sort_by { |file_path, duration| file_path }.each do |file_path, duration|
+          file.puts "#{file_path},#{duration}"
+        end
+      end
+    end
+
+    def self.current_time_millis
+      (Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000).to_i
+    end
+  end
+end

data/rspec-sharder.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "lib/rspec-sharder/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "rspec-sharder"
+  spec.version       = RSpec::Sharder::VERSION
+  spec.authors       = ["Nick Dower"]
+  spec.email         = ["nicholasdower@gmail.com"]
+
+  spec.summary       = "A utility which shards specs."
+  spec.description   = "A utility which shards specs."
+  spec.homepage      = "https://github.com/nicholasdower/rspec-sharder"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/nicholasdower/rspec-sharder"
+  spec.metadata["changelog_uri"] = "https://github.com/nicholasdower/rspec-sharder/releases"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = 'bin'
+  spec.executables   << 'rspec-sharder'
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'rspec-core'
+end

data/scripts/release.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+if [ $# -ne 1 ]; then
+  echo "usage: $0 <version>" >&2
+  exit 1
+fi
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "error: stage or commit your changes." >&2
+  exit 1;
+fi
+
+NEW_VERSION=$1
+CURRENT_VERSION=$(grep VERSION lib/rspec-sharder/version.rb | cut -d'"' -f 2)
+
+echo "Updating from v$CURRENT_VERSION to v$NEW_VERSION. Press enter to continue."
+read
+
+sed -E -i '' "s/VERSION = \"[^\"]+\"/VERSION = \"$NEW_VERSION\"/g" lib/rspec-sharder/version.rb
+gem build
+gem push rspec-sharder-$NEW_VERSION.gem
+bundle install
+git commit -a -m "v$NEW_VERSION Release"
+open "https://github.com/nicholasdower/rspec-sharder/releases/new?title=v$NEW_VERSION%20Release&tag=v$NEW_VERSION&target=$(git rev-parse HEAD)"

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: rspec-sharder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Nick Dower
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-08-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A utility which shards specs.
+email:
+- nicholasdower@gmail.com
+executables:
+- rspec-sharder
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- bin/rspec-sharder
+- lib/rspec-sharder.rb
+- lib/rspec-sharder/version.rb
+- rspec-sharder.gemspec
+- scripts/release.sh
+homepage: https://github.com/nicholasdower/rspec-sharder
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/nicholasdower/rspec-sharder
+  source_code_uri: https://github.com/nicholasdower/rspec-sharder
+  changelog_uri: https://github.com/nicholasdower/rspec-sharder/releases
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: A utility which shards specs.
+test_files: []