ci-queue 0.11.1 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a44623e2359645e895ede304d4499f5efb7c689b
-  data.tar.gz: 764685c0bb4dc8b568455aa3edb0ae7a384f5893
+SHA256:
+  metadata.gz: 78b51045e571e242b8d4b64e49f37abd462be948f007de6a63380ca0f448ce4c
+  data.tar.gz: bdec2201394129e091516b82e5709a016209aec6a8f41488132ac8400c88150d
 SHA512:
-  metadata.gz: 34f7fd1d6196c1b68e850b06472d8961acd449fd68cee006a0cfc9e6b2d79b338a22aad78a6e22efeba79d424b8eeaa595c857d81d63470e157ebcf7e7a1404e
-  data.tar.gz: 641f5fb0cc38c5adbfe6308d637d8a0f93b66c2a1e547ff87d7077147f2d3edc0b818c6d15298e076f66a2b050878aa4f9778880c671d6a96094b9c11ab17ce5
+  metadata.gz: e51188516bdd8b04c29981a4cd7f89ac2f6a8a7979c72210065362d34b86565522e3e26305a2c088656d543c064c802c4886f6ad4801f675b7c73cbec82da38c
+  data.tar.gz: 14853c01858461192992d2dd330a96dcb66ca6a9c17100c4d73b4d6b364cb7b1c7e8661b14d8136a454f2d0bf0a72a8af375a07dd6eba91a2541784f6964f1fc

data/.gitignore

@@ -1,3 +1,5 @@
+test/fixtures/log/junit.xml
+test/fixtures/log/test_order.log
 /.byebug_history
 /.bundle/
 /.yardoc

data/dev.yml

@@ -9,12 +9,3 @@ up:
 
 commands:
   test: REDIS_HOST=ci-queue.railgun bundle exec rake test
-
-railgun:
-  image: dev:railgun-common-services-0.2.x
-  services:
-    redis: 6379
-  ip_address: 192.168.64.245
-  memory: 1G
-  cores: 1
-  disk: 512M

data/lib/ci/queue/configuration.rb

@@ -1,3 +1,4 @@
+
 module CI
   module Queue
     class Configuration
@@ -9,13 +10,21 @@ module CI
             build_id: env['CIRCLE_BUILD_URL'] || env['BUILDKITE_BUILD_ID'] || env['TRAVIS_BUILD_ID'],
             worker_id: env['CIRCLE_NODE_INDEX'] || env['BUILDKITE_PARALLEL_JOB'],
             seed: env['CIRCLE_SHA1'] || env['BUILDKITE_COMMIT'] || env['TRAVIS_COMMIT'],
+            flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
           )
         end
+
+        def load_flaky_tests(path)
+          return [] unless path
+          ::File.readlines(path).map(&:chomp).to_set
+        rescue SystemCallError
+          []
+        end
       end
 
       def initialize(
         timeout: 30, build_id: nil, worker_id: nil, max_requeues: 0, requeue_tolerance: 0,
-        namespace: nil, seed: nil
+        namespace: nil, seed: nil, flaky_tests: []
       )
         @namespace = namespace
         @timeout = timeout
@@ -24,6 +33,11 @@ module CI
         @max_requeues = max_requeues
         @requeue_tolerance = requeue_tolerance
         @seed = seed
+        @flaky_tests = flaky_tests
+      end
+
+      def flaky?(test)
+        @flaky_tests.include?(test.id)
       end
 
       def seed

data/lib/ci/queue/redis/base.rb

@@ -8,6 +8,10 @@ module CI
           @config = config
         end
 
+        def flaky?(test)
+          @config.flaky?(test)
+        end
+
         def exhausted?
           queue_initialized? && size == 0
         end

data/lib/ci/queue/static.rb

@@ -17,6 +17,10 @@ module CI
         @total = tests.size
       end
 
+      def flaky?(test)
+        @config.flaky?(test)
+      end
+
       def build
         @build ||= BuildRecord.new(self)
       end

data/lib/ci/queue/version.rb

@@ -1,6 +1,6 @@
 module CI
   module Queue
-    VERSION = '0.11.1'
+    VERSION = '0.12.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue.rb

@@ -7,7 +7,8 @@ require 'minitest/queue/error_report'
 require 'minitest/queue/local_requeue_reporter'
 require 'minitest/queue/build_status_recorder'
 require 'minitest/queue/build_status_reporter'
-
+require 'minitest/queue/order_reporter'
+require 'minitest/queue/junit_reporter'
 
 module Minitest
   class Requeue < Skip
@@ -35,6 +36,31 @@ module Minitest
     end
   end
 
+  class Flaked < Skip
+    attr_reader :failure
+
+    def initialize(failure)
+      super()
+      @failure = failure
+    end
+
+    def result_label
+      "Skipped"
+    end
+
+    def backtrace
+      failure.backtrace
+    end
+
+    def error
+      failure.error
+    end
+
+    def message
+      failure.message
+    end
+  end
+
   module Requeueing
     # Make requeues acts as skips for reporters not aware of the difference.
     def skipped?
@@ -50,6 +76,25 @@ module Minitest
     end
   end
 
+  module Flakiness
+    # Make failed flaky tests acts as skips for reporters not aware of the difference.
+    def skipped?
+      super || flaked?
+    end
+
+    def flaked?
+      !!((Flaked === failure) || @flaky)
+    end
+
+    def mark_as_flaked!
+      if passed?
+        @flaky = true
+      else
+        self.failures.unshift(Flaked.new(self.failures.shift))
+      end
+    end
+  end
+
   module Queue
     class SingleExample
       def initialize(runnable, method_name)
@@ -68,6 +113,10 @@ module Minitest
       def run
         Minitest.run_one_method(@runnable, @method_name)
       end
+
+      def flaky?
+        Minitest.queue.flaky?(self)
+      end
     end
 
     attr_reader :queue
@@ -103,6 +152,12 @@ module Minitest
       queue.poll do |example|
         result = example.run
         failed = !(result.passed? || result.skipped?)
+
+        if example.flaky?
+          result.mark_as_flaked!
+          failed = false
+        end
+
         if failed && queue.requeue(example)
           result.requeue!
           reporter.record(result)
@@ -119,7 +174,19 @@ end
 MiniTest.singleton_class.prepend(MiniTest::Queue)
 if defined? MiniTest::Result
   MiniTest::Result.prepend(MiniTest::Requeueing)
+  MiniTest::Result.prepend(MiniTest::Flakiness)
 else
   MiniTest::Test.prepend(MiniTest::Requeueing)
-  MiniTest::Test.send(:alias_method, :klass, :class)
+  MiniTest::Test.prepend(MiniTest::Flakiness)
+
+  module MinitestBackwardCompatibility
+    def source_location
+      method(name).source_location
+    end
+
+    def klass
+      self.class.name
+    end
+  end
+  MiniTest::Test.prepend(MinitestBackwardCompatibility)
 end

data/lib/minitest/queue/junit_reporter.rb

@@ -0,0 +1,113 @@
+require 'minitest/reporters'
+require 'builder'
+require 'fileutils'
+
+module Minitest
+  module Queue
+    class JUnitReporter < Minitest::Reporters::BaseReporter
+      class XmlMarkup < ::Builder::XmlMarkup
+        def trunc!(txt)
+          txt.sub(/\n.*/m, '...')
+        end
+      end
+
+      def initialize(report_path = 'log/junit.xml', options = {})
+        super({})
+        @report_path = File.absolute_path(report_path)
+        @base_path = options[:base_path] || Dir.pwd
+      end
+
+      def report
+        super
+
+        suites = tests.group_by { |test| test.klass }
+
+        xml = Builder::XmlMarkup.new(indent: 2)
+        xml.instruct!
+        xml.test_suites do
+          suites.each do |suite, tests|
+            add_tests_to(xml, suite, tests)
+          end
+        end
+        FileUtils.mkdir_p(File.dirname(@report_path))
+        File.open(@report_path, 'w+') { |file| file << xml.target! }
+      end
+
+      private
+
+      def add_tests_to(xml, suite, tests)
+        suite_result = analyze_suite(tests)
+        file_path = Pathname.new(tests.first.source_location.first)
+        base_path = Pathname.new(@base_path)
+        relative_path = file_path.relative_path_from(base_path)
+
+        xml.testsuite(name: suite, filepath: relative_path,
+                      skipped: suite_result[:skip_count], failures: suite_result[:fail_count],
+                      errors: suite_result[:error_count], tests: suite_result[:test_count],
+                      assertions: suite_result[:assertion_count], time: suite_result[:time]) do
+          tests.each do |test|
+            lineno = test.source_location.last
+            xml.testcase(name: test.name, lineno: lineno, classname: suite, assertions: test.assertions,
+                         time: test.time, flaky_test: test.flaked?) do
+              xml << xml_message_for(test) unless test.passed?
+            end
+          end
+        end
+      end
+
+      def xml_message_for(test)
+        xml = XmlMarkup.new(indent: 2, margin: 2)
+        error = test.failure
+
+        if test.skipped? && !test.flaked?
+          xml.skipped(type: test.name)
+        elsif test.error?
+          xml.error(type: test.name, message: xml.trunc!(error.message)) do
+            xml.text!(message_for(test))
+          end
+        elsif test.failure
+          xml.failure(type: test.name, message: xml.trunc!(error.message)) do
+            xml.text!(message_for(test))
+          end
+        end
+      end
+
+      def message_for(test)
+        suite = test.klass
+        name = test.name
+        error = test.failure
+
+        if test.passed?
+          nil
+        elsif test.skipped?
+          "Skipped:\n#{name}(#{suite}) [#{location(error)}]:\n#{error.message}\n"
+        elsif test.failure
+          "Failure:\n#{name}(#{suite}) [#{location(error)}]:\n#{error.message}\n"
+        elsif test.error?
+          "Error:\n#{name}(#{suite}):\n#{error.message}"
+        end
+      end
+
+      def location(exception)
+        last_before_assertion = ''
+        exception.backtrace.reverse_each do |s|
+          break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/
+          last_before_assertion = s
+        end
+        last_before_assertion.sub(/:in .*$/, '')
+      end
+
+      def analyze_suite(tests)
+        result = Hash.new(0)
+        result[:time] = 0
+        tests.each do |test|
+          result[:"#{result(test)}_count"] += 1
+          result[:assertion_count] += test.assertions
+          result[:test_count] += 1
+          result[:time] += test.time
+        end
+        result
+      end
+    end
+  end
+end

data/lib/minitest/{reporters → queue}/order_reporter.rb

@@ -1,6 +1,6 @@
 require 'minitest/reporters'
 
-class Minitest::Reporters::OrderReporter < Minitest::Reporters::BaseReporter
+class Minitest::Queue::OrderReporter < Minitest::Reporters::BaseReporter
   def initialize(options = {})
     @path = options.delete(:path)
     super

data/lib/minitest/queue/runner.rb

@@ -44,8 +44,10 @@ module Minitest
         set_load_path
         Minitest.queue = queue
         Minitest.queue_reporters = [
-          Minitest::Queue::LocalRequeueReporter.new,
-          Minitest::Queue::BuildStatusRecorder.new(build: queue.build)
+          LocalRequeueReporter.new,
+          BuildStatusRecorder.new(build: queue.build),
+          JUnitReporter.new,
+          OrderReporter.new(path: 'log/test_order.log'),
         ]
 
         trap('TERM') { Minitest.queue.shutdown! }

data/railgun.yml

@@ -0,0 +1,13 @@
+# This file is for Shopify employees development environment.
+# If you are an external contributor you don't have to bother with it.
+name: ci-queue
+
+vm:
+  image: /opt/dev/misc/railgun-images/default
+  ip_address: 192.168.64.245
+  memory: 1G
+  cores: 2
+volumes:
+  root: 512M
+services:
+   - redis

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-13 00:00:00.000000000 Z
+date: 2018-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi
@@ -194,13 +194,14 @@ files:
 - lib/minitest/queue/build_status_reporter.rb
 - lib/minitest/queue/error_report.rb
 - lib/minitest/queue/failure_formatter.rb
+- lib/minitest/queue/junit_reporter.rb
 - lib/minitest/queue/local_requeue_reporter.rb
+- lib/minitest/queue/order_reporter.rb
 - lib/minitest/queue/runner.rb
 - lib/minitest/reporters/bisect_reporter.rb
-- lib/minitest/reporters/order_reporter.rb
-- lib/minitest/reporters/redis_reporter.rb
 - lib/rspec/queue.rb
 - lib/rspec/queue/build_status_recorder.rb
+- railgun.yml
 homepage: https://github.com/Shopify/ci-queue
 licenses:
 - MIT
@@ -221,7 +222,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue

data/lib/minitest/reporters/redis_reporter.rb

@@ -1,91 +0,0 @@
-require 'minitest/reporters'
-
-module Minitest
-  module Reporters
-    module RedisReporter
-      include ANSI::Code
-
-      COUNTERS = %w(
-        assertions
-        errors
-        failures
-        skips
-        requeues
-        total_time
-      ).freeze
-
-      class << self
-        attr_accessor :failure_formatter
-      end
-      self.failure_formatter = FailureFormatter
-
-      class Base < BaseReporter
-        def initialize(build:, **options)
-          @build = build
-          super(options)
-        end
-
-        def completed?
-          build.queue_exhausted?
-        end
-
-        def error_reports
-          build.error_reports.sort_by(&:first).map { |k, v| ErrorReport.load(v) }
-        end
-
-        private
-
-      end
-
-      class Summary < Base
-      end
-
-      class Worker < Base
-        attr_accessor :requeues
-
-        def initialize(*)
-          super
-          self.failures = 0
-          self.errors = 0
-          self.skips = 0
-          self.requeues = 0
-        end
-
-        def report
-          # noop
-        end
-
-        def record(test)
-          super
-
-          self.total_time = Minitest.clock_time - start_time
-          if test.requeued?
-            self.requeues += 1
-          elsif test.skipped?
-            self.skips += 1
-          elsif test.error?
-            self.errors += 1
-          elsif test.failure
-            self.failures += 1
-          end
-
-
-          stats = COUNTERS.zip(COUNTERS.map { |c| send(c) })
-          if (test.failure || test.error?) && !test.skipped?
-            build.record_error("#{test.klass}##{test.name}", dump(test), stats: stats)
-          else
-            build.record_success("#{test.klass}##{test.name}", stats: stats)
-          end
-        end
-
-        private
-
-        def dump(test)
-          ErrorReport.new(RedisReporter.failure_formatter.new(test).to_h).dump
-        end
-
-        attr_reader :aggregates
-      end
-    end
-  end
-end