ci-queue 0.16.0 → 0.17.0

data/lib/minitest/queue/statsd.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Implements a small and limited StatsD implementation to reduce importing unnecessary dependencies because
 # we don't want to require on the bundle which would slow down a CI Queue run
 

data/lib/minitest/queue/test_data.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+require 'minitest/reporters'
+require 'fileutils'
+
+module Minitest
+  module Queue
+    class TestData
+      attr_reader :namespace, :test_index
+
+      def initialize(test:, index:, namespace:, base_path:)
+        @test = test
+        @base_path = base_path
+        @namespace = namespace
+        @test_index = index
+      end
+
+      def test_id
+        "#{test_suite}##{test_name}"
+      end
+
+      def test_name
+        @test.name
+      end
+
+      def test_suite
+        @test.klass
+      end
+
+      def test_retried
+        @test.requeued?
+      end
+
+      def test_result
+        if @test.passed?
+          'success'
+        elsif !@test.requeued? && @test.skipped?
+          'skipped'
+        elsif @test.error?
+          'error'
+        elsif @test.failure
+          'failure'
+        else
+          'undefined'
+        end
+      end
+
+      def test_assertions
+        @test.assertions
+      end
+
+      def test_duration
+        @test.time
+      end
+
+      def test_file_path
+        path = @test.source_location.first
+        relative_path_for(path)
+      end
+
+      def test_file_line_number
+        @test.source_location.last
+      end
+
+      # Error class only considers failures wheras the other error fields also consider skips
+      def error_class
+        return nil unless @test.failure
+
+        @test.failure.error.class.name
+      end
+
+      def error_message
+        return nil unless @test.failure
+
+        @test.failure.message
+      end
+
+      def error_file_path
+        return nil unless @test.failure
+
+        path = error_location(@test.failure).first
+        relative_path_for(path)
+      end
+
+      def error_file_number
+        return nil unless @test.failure
+
+        error_location(@test.failure).last
+      end
+
+      def to_h
+        {
+          namespace: namespace,
+          test_id: test_id,
+          test_name: test_name,
+          test_suite: test_suite,
+          test_result: test_result,
+          test_index: test_index,
+          test_result_ignored: @test.flaked?,
+          test_retried: test_retried,
+          test_assertions: test_assertions,
+          test_duration: test_duration,
+          test_file_path: test_file_path,
+          test_file_line_number: test_file_line_number,
+          error_class: error_class,
+          error_message: error_message,
+          error_file_path: error_file_path,
+          error_file_number: error_file_number,
+        }
+      end
+
+      private
+
+      def relative_path_for(path)
+        file_path = Pathname.new(path)
+        base_path = Pathname.new(@base_path)
+        file_path.relative_path_from(base_path)
+      end
+
+      def error_location(exception)
+        @error_location ||= begin
+          last_before_assertion = ''
+          exception.backtrace.reverse_each do |s|
+            break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/
+
+            last_before_assertion = s
+          end
+          path = last_before_assertion.sub(/:in .*$/, '')
+          # the path includes the linenumber at the end,
+          # which is seperated by a :
+          # rpartition splits the string at the last occurence of :
+          result = path.rpartition(':')
+          # We return [path, linenumber] here
+          [result.first, result.last.to_i]
+        end
+      end
+    end
+  end
+end

data/lib/minitest/queue/test_data_reporter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+require 'minitest/reporters'
+require 'fileutils'
+require 'json'
+
+require 'minitest/queue/test_data'
+
+module Minitest
+  module Queue
+    class TestDataReporter < Minitest::Reporters::BaseReporter
+      def initialize(report_path: 'log/test_data.json', base_path: nil, namespace: '')
+        super({})
+        @report_path = File.absolute_path(report_path)
+        @base_path = base_path || Dir.pwd
+        @namespace = namespace || ''
+      end
+
+      def report
+        super
+
+        result = tests.map.with_index do |test, index|
+          Queue::TestData.new(test: test, index: index,
+                              base_path: @base_path, namespace: @namespace).to_h
+        end.to_json
+
+        dirname = File.dirname(@report_path)
+        FileUtils.mkdir_p(dirname)
+        File.open(@report_path, 'w+') { |file| file << result }
+      end
+    end
+  end
+end

data/lib/minitest/queue/test_time_recorder.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+module Minitest
+  module Queue
+    class TestTimeRecorder < Minitest::Reporters::BaseReporter
+      def initialize(build:, **options)
+        super(options)
+        @build = build
+      end
+
+      def record(test)
+        return unless test.passed?
+        test_duration_in_milliseconds = test.time * 1000
+        @build.record(test.name, test_duration_in_milliseconds)
+      end
+    end
+  end
+end

data/lib/minitest/queue/test_time_reporter.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+require 'minitest/reporters'
+
+module Minitest
+  module Queue
+    class TestTimeReporter < Minitest::Reporters::BaseReporter
+      include ::CI::Queue::OutputHelpers
+
+      def initialize(build:, limit: nil, percentile: nil, **options)
+        super(options)
+        @test_time_hash = build.fetch
+        @limit = limit
+        @percentile = percentile
+        @success = true
+      end
+
+      def report
+        return if limit.nil? || test_time_hash.empty?
+
+        puts '+++ Test Time Report'
+
+        if offending_tests.empty?
+          msg = "The #{humanized_percentile} of test execution time is within #{limit} milliseconds."
+          puts green(msg)
+          return
+        end
+
+        @success = false
+        puts <<~EOS
+          #{red("Detected #{offending_tests.size} test(s) over the desired time limit.")}
+          Please make them faster than #{limit}ms in the #{humanized_percentile} percentile.
+        EOS
+        offending_tests.each do |test_name, duration|
+          puts "#{red(test_name)}: #{duration}ms"
+        end
+      end
+
+      def success?
+        @success
+      end
+
+      def record(*)
+        raise NotImplementedError
+      end
+
+      private
+
+      attr_reader :test_time_hash, :limit, :percentile
+
+      def humanized_percentile
+        percentile_in_percentage = percentile * 100
+        "#{percentile_in_percentage.to_i}th"
+      end
+
+      def offending_tests
+        @offending_tests ||= begin
+          test_time_hash.each_with_object({}) do |(test_name, durations), offenders|
+            duration = calculate_percentile(durations)
+            next if duration <= limit
+            offenders[test_name] = duration
+          end
+        end
+      end
+
+      def calculate_percentile(array)
+        array.sort[(percentile * array.length).ceil - 1]
+      end
+    end
+  end
+end

data/lib/minitest/reporters/bisect_reporter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'minitest/reporters'
 
 module Minitest

data/lib/minitest/reporters/statsd_reporter.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'minitest/queue'
 require 'minitest/queue/statsd'
 require 'minitest/reporters'

data/lib/rspec/queue.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
+require 'fileutils'
 require 'delegate'
 require 'rspec/core'
 require 'ci/queue'
 require 'rspec/queue/build_status_recorder'
+require 'rspec/queue/order_recorder'
 
 module RSpec
   module Queue
@@ -64,7 +67,7 @@ module RSpec
 
         parser.separator("\n  **** Queue options ****\n\n")
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           URL of the queue, e.g. redis://example.com.
           Defaults to $CI_QUEUE_URL if set.
         EOS
@@ -73,7 +76,7 @@ module RSpec
           options[:queue_url] = url
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Wait for all workers to complete and summarize the test failures.
         EOS
         parser.on('--report', *help) do |url|
@@ -81,14 +84,14 @@ module RSpec
           options[:runner] = RSpec::Queue::ReportRunner.new
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Replays a previous run in the same order.
         EOS
         parser.on('--retry', *help) do |url|
           STDERR.puts "Warning: The --retry flag is deprecated"
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Unique identifier for the workload. All workers working on the same suite of tests must have the same build identifier.
           If the build is tried again, or another revision is built, this value must be different.
           It's automatically inferred on Buildkite, CircleCI and Travis.
@@ -98,7 +101,7 @@ module RSpec
           queue_config.build_id = build_id
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Optional. Sets a prefix for the build id in case a single CI build runs multiple independent test suites.
             Example: --namespace integration
         EOS
@@ -107,7 +110,7 @@ module RSpec
           queue_config.namespace = namespace
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Specify a timeout after which if a test haven't completed, it will be picked up by another worker.
           It is very important to set this vlaue higher than the slowest test in the suite, otherwise performance will be impacted.
           Defaults to 30 seconds.
@@ -117,7 +120,7 @@ module RSpec
           queue_config.timeout = Float(timeout)
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           A unique identifier for this worker, It must be consistent to allow retries.
           If not specified, retries won't be available.
           It's automatically inferred on Buildkite and CircleCI.
@@ -127,7 +130,7 @@ module RSpec
           queue_config.worker_id = worker_id
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Defines how many time a single test can be requeued.
           Defaults to 0.
         EOS
@@ -136,7 +139,7 @@ module RSpec
           queue_config.max_requeues = Integer(max)
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Defines how many requeues can happen overall, based on the test suite size. e.g 0.05 for 5%.
           Defaults to 0.
         EOS
@@ -145,7 +148,7 @@ module RSpec
           queue_config.requeue_tolerance = Float(ratio)
         end
 
-        help = split_heredoc(<<-EOS)
+        help = <<~EOS
           Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
           Defaults to disabled.
         EOS
@@ -157,10 +160,6 @@ module RSpec
         parser
       end
 
-      def split_heredoc(string)
-        string.lines.map(&:strip)
-      end
-
       def queue_config
         ::RSpec::Queue.config
       end
@@ -387,6 +386,8 @@ module RSpec
         success = true
         @configuration.reporter.report(examples_count) do |reporter|
           @configuration.add_formatter(BuildStatusRecorder)
+          FileUtils.mkdir_p('log')
+          @configuration.add_formatter(OrderRecorder, open('log/test_order.log', 'w+'))
 
           @configuration.with_suite_hooks do
             break if @world.wants_to_quit

data/lib/rspec/queue/build_status_recorder.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module RSpec
   module Queue
     class BuildStatusRecorder

data/lib/rspec/queue/order_recorder.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+require 'rspec/core/formatters/base_formatter'
+
+module RSpec
+  module Queue
+    class OrderRecorder < ::RSpec::Core::Formatters::BaseFormatter
+      ::RSpec::Core::Formatters.register self, :example_started
+
+      def initialize(*)
+        super
+        output.sync = true
+      end
+
+      def example_started(notification)
+        return if notification.is_a?(RSpec::Core::Notifications::SkippedExampleNotification)
+        output.write("#{notification.example.id}\n")
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-19 00:00:00.000000000 Z
+date: 2020-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: To parallelize your CI without having to balance your tests
 email:
 - jean.boussier@shopify.com
@@ -178,16 +192,21 @@ files:
 - lib/ci/queue/common.rb
 - lib/ci/queue/configuration.rb
 - lib/ci/queue/file.rb
+- lib/ci/queue/grind.rb
 - lib/ci/queue/output_helpers.rb
 - lib/ci/queue/redis.rb
 - lib/ci/queue/redis/acknowledge.lua
 - lib/ci/queue/redis/base.rb
 - lib/ci/queue/redis/build_record.rb
+- lib/ci/queue/redis/grind.rb
+- lib/ci/queue/redis/grind_record.rb
+- lib/ci/queue/redis/grind_supervisor.rb
 - lib/ci/queue/redis/requeue.lua
 - lib/ci/queue/redis/reserve.lua
 - lib/ci/queue/redis/reserve_lost.lua
 - lib/ci/queue/redis/retry.rb
 - lib/ci/queue/redis/supervisor.rb
+- lib/ci/queue/redis/test_time_record.rb
 - lib/ci/queue/redis/worker.rb
 - lib/ci/queue/static.rb
 - lib/ci/queue/version.rb
@@ -196,15 +215,22 @@ files:
 - lib/minitest/queue/build_status_reporter.rb
 - lib/minitest/queue/error_report.rb
 - lib/minitest/queue/failure_formatter.rb
+- lib/minitest/queue/grind_recorder.rb
+- lib/minitest/queue/grind_reporter.rb
 - lib/minitest/queue/junit_reporter.rb
 - lib/minitest/queue/local_requeue_reporter.rb
 - lib/minitest/queue/order_reporter.rb
 - lib/minitest/queue/runner.rb
 - lib/minitest/queue/statsd.rb
+- lib/minitest/queue/test_data.rb
+- lib/minitest/queue/test_data_reporter.rb
+- lib/minitest/queue/test_time_recorder.rb
+- lib/minitest/queue/test_time_reporter.rb
 - lib/minitest/reporters/bisect_reporter.rb
 - lib/minitest/reporters/statsd_reporter.rb
 - lib/rspec/queue.rb
 - lib/rspec/queue/build_status_recorder.rb
+- lib/rspec/queue/order_recorder.rb
 - railgun.yml
 homepage: https://github.com/Shopify/ci-queue
 licenses:
@@ -225,8 +251,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue