ci-queue 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11fda6e3d6ac119631a90d63086ebb9e06d628bd
-  data.tar.gz: 9ecad3efc2ec99b01e5a58f9defc55b772ffa65b
+  metadata.gz: 61155b4429b71670e473e39cf41dbac081c24727
+  data.tar.gz: 60d3c11057a5d7412554f51f10fe8beeefb4a483
 SHA512:
-  metadata.gz: 121adc5f59a3c95c62121e6bad8ebeb321890be2a67e3f3ae14ae5b6fd5febab786b8d4a4bd4e74e54e14bc29968dc07cc3a2c54d30c7230e83a1b52cfdfb58f
-  data.tar.gz: 658a17f26818a69bb7a60c9e563a5b2e16dc4dded98637455e49313d438b3d72fa323b1a10c0179d223c8f32b95009238cddc6e145c7db46754a7114ec596ec8
+  metadata.gz: d59e89e8a27f00177f32ee55d7115b93617e21a9cfa2161a211da558831aeddadd4739242a93e9f12e183458839e6766903084b03814730848127a5febd9e21b
+  data.tar.gz: 377c2123f5a51743a78abd13b5fd5c72f3bada7a1f230ea6712cef43275365e03a166f54c01097d111f2fd390b8f71d756732768fb70cc711d40c087867ee804

data/.gitignore

@@ -1,3 +1,4 @@
+/.byebug_history
 /.bundle/
 /.yardoc
 /Gemfile.lock

data/.travis.yml

@@ -1,5 +1,7 @@
 sudo: false
+services:
+  - redis-server
 language: ruby
 rvm:
   - 2.3.1
-before_install: gem install bundler -v 1.13.3
+before_install: gem install bundler -v 1.13.6

data/README.md

@@ -1,5 +1,8 @@
 # CI::Queue
 
+[![Gem Version](https://badge.fury.io/rb/ci-queue.svg)](https://rubygems.org/gems/ci-queue)
+[![Build Status](https://travis-ci.org/Shopify/ci-queue.svg?branch=master)](https://travis-ci.org/Shopify/ci-queue)
+
 Distribute tests over many workers using a queue. 
 
 ## Why a queue?

data/Rakefile

@@ -4,7 +4,7 @@ require 'rake/testtask'
 Rake::TestTask.new(:test) do |t|
   t.libs << 'test'
   t.libs << 'lib'
-  t.test_files = FileList['test/**/*_test.rb']
+  t.test_files = FileList['test/**/*_test.rb'] - FileList['test/fixtures/*_test.rb']
 end
 
 task :default => :test

data/ci-queue.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'minitest', '~> 5.0'
   spec.add_development_dependency 'redis', '~> 3.3'
   spec.add_development_dependency 'simplecov', '~> 0.12'
+  spec.add_development_dependency 'minitest-reporters', '~> 1.1'
 end

data/lib/ci/queue/file.rb

@@ -3,8 +3,8 @@ require 'ci/queue/static'
 module CI
   module Queue
     class File < Static
-      def initialize(path)
-        super(::File.readlines(path).map(&:strip).reject(&:empty?))
+      def initialize(path, **args)
+        super(::File.readlines(path).map(&:strip).reject(&:empty?), **args)
       end
     end
   end

data/lib/ci/queue/redis.rb

@@ -1,6 +1,7 @@
 require 'redis'
 require 'ci/queue/redis/base'
 require 'ci/queue/redis/worker'
+require 'ci/queue/redis/retry'
 require 'ci/queue/redis/supervisor'
 
 module CI

data/lib/ci/queue/redis/base.rb

@@ -4,7 +4,7 @@ module CI
       class Base
         def initialize(redis:, build_id:)
           @redis = redis
-          @key = "build:#{build_id}"
+          @build_id = build_id
         end
 
         def empty?
@@ -44,10 +44,10 @@ module CI
 
         private
 
-        attr_reader :redis
+        attr_reader :redis, :build_id
 
         def key(*args)
-          [@key, *args].join(':')
+          ['build', build_id, *args].join(':')
         end
 
         def master_status
@@ -55,9 +55,12 @@ module CI
         end
 
         def eval_script(script, *args)
+          redis.evalsha(load_script(script), *args)
+        end
+
+        def load_script(script)
           @scripts_cache ||= {}
-          sha = (@scripts_cache[script] ||= redis.script(:load, script))
-          redis.evalsha(sha, *args)
+          @scripts_cache[script] ||= redis.script(:load, script)
         end
       end
     end

data/lib/ci/queue/redis/retry.rb

@@ -0,0 +1,29 @@
+module CI
+  module Queue
+    module Redis
+      class Retry < Static
+        def initialize(tests, redis:, build_id:, worker_id:, **args)
+          @redis = redis
+          @build_id = build_id
+          @worker_id = worker_id
+          super(tests, **args)
+        end
+
+        def minitest_reporters
+          require 'minitest/reporters/redis_reporter'
+          @minitest_reporters ||= [
+            Minitest::Reporters::RedisReporter::Worker.new(
+              redis: redis,
+              build_id: build_id,
+              worker_id: worker_id,
+            )
+          ]
+        end
+
+        private
+
+        attr_reader :redis, :build_id, :worker_id
+      end
+    end
+  end
+end

data/lib/ci/queue/redis/worker.rb

@@ -1,10 +1,17 @@
+require 'ci/queue/static'
+
 module CI
   module Queue
     module Redis
+      ReservationError = Class.new(StandardError)
+
       class Worker < Base
         attr_reader :total
 
-        def initialize(tests, redis:, build_id:, worker_id:, timeout:)
+        def initialize(tests, redis:, build_id:, worker_id:, timeout:, max_requeues: 0, requeue_tolerance: 0.0)
+          @reserved_test = nil
+          @max_requeues = max_requeues
+          @global_max_requeues = (tests.size * requeue_tolerance).ceil
           @shutdown_required = false
           super(redis: redis, build_id: build_id)
           @worker_id = worker_id
@@ -26,19 +33,90 @@ module CI
 
         def poll
           wait_for_master
-          while test = reserve
-            yield test
-            acknowledge(test)
+          until shutdown_required? || empty?
+            if test = reserve
+              yield test
+            else
+              sleep 0.05
+            end
           end
         end
 
-        def retry_queue
-          Static.new(redis.lrange(key("worker:#{worker_id}:queue"), 0, -1).reverse)
+        def retry_queue(**args)
+          Retry.new(
+            redis.lrange(key('worker', worker_id, 'queue'), 0, -1).reverse.uniq,
+            redis: redis,
+            build_id: build_id,
+            worker_id: worker_id,
+            **args
+          )
+        end
+
+        def minitest_reporters
+          require 'minitest/reporters/redis_reporter'
+          @minitest_reporters ||= [
+            Minitest::Reporters::RedisReporter::Worker.new(
+              redis: redis,
+              build_id: build_id,
+              worker_id: worker_id,
+            )
+          ]
+        end
+
+        def acknowledge(test, success)
+          if @reserved_test == test
+            @reserved_test = nil
+          else
+            raise ReservationError, "Acknowledged #{test.inspect} but #{@reserved_test.inspect} was reserved"
+          end
+
+          if !success && should_requeue?(test)
+            requeue(test)
+            false
+          else
+            ack(test)
+            true
+          end
         end
 
         private
 
-        attr_reader :worker_id, :timeout
+        attr_reader :worker_id, :timeout, :max_requeues, :global_max_requeues
+
+        def should_requeue?(test)
+          individual_requeues, global_requeues = redis.multi do
+            redis.hincrby(key('requeues-count'), test, 1)
+            redis.hincrby(key('requeues-count'), '___total___'.freeze, 1)
+          end
+
+          if individual_requeues.to_i > max_requeues || global_requeues.to_i > global_max_requeues
+            redis.multi do
+              redis.hincrby(key('requeues-count'), test, -1)
+              redis.hincrby(key('requeues-count'), '___total___'.freeze, -1)
+            end
+            return false
+          end
+
+          true
+        end
+
+        def requeue(test)
+          load_script(ACKNOWLEDGE)
+          redis.multi do
+            redis.decr(key('processed'))
+            redis.rpush(key('queue'), test)
+            ack(test)
+          end
+        end
+
+        def reserve
+          if @reserved_test
+            raise ReservationError, "#{@reserved_test.inspect} is already reserved. " \
+              "You have to acknowledge it before you can reserve another one"
+          end
+
+          @reserved_test = (try_to_reserve_lost_test || try_to_reserve_test)
+        end
 
         RESERVE_TEST = %{
           local queue_key = KEYS[1]
@@ -53,14 +131,8 @@ module CI
             return nil
           end
         }
-        def reserve
-          return if shutdown_required?
-
-          if test = eval_script(RESERVE_TEST, keys: [key('queue'), key('running')], argv: [Time.now.to_f])
-            return test
-          else
-            reserve_lost_test
-          end
+        def try_to_reserve_test
+          eval_script(RESERVE_TEST, keys: [key('queue'), key('running')], argv: [Time.now.to_f])
         end
 
         RESERVE_LOST_TEST = %{
@@ -76,14 +148,8 @@ module CI
             return nil
           end
         }
-        def reserve_lost_test
-          until redis.zcard(key('running')) == 0
-            if test = eval_script(RESERVE_LOST_TEST, keys: [key('running')], argv: [Time.now.to_f, timeout])
-              return test
-            end
-            sleep 0.1
-          end
-          nil
+        def try_to_reserve_lost_test
+          eval_script(RESERVE_LOST_TEST, keys: [key('running')], argv: [Time.now.to_f, timeout])
         end
 
         ACKNOWLEDGE = %{
@@ -95,9 +161,9 @@ module CI
             redis.call('incr', processed_count_key)
           end
         }
-        def acknowledge(test)
+        def ack(test)
           eval_script(ACKNOWLEDGE, keys: [key('running'), key('processed')], argv: [test])
-          redis.lpush(key("worker:#{worker_id}:queue"), test)
+          redis.lpush(key('worker', worker_id, 'queue'), test)
         end
 
         def push(tests)

data/lib/ci/queue/static.rb

@@ -3,10 +3,12 @@ module CI
     class Static
       attr_reader :progress, :total
 
-      def initialize(tests)
+      def initialize(tests, max_requeues: 0, requeue_tolerance: 0.0)
         @queue = tests
         @progress = 0
         @total = tests.size
+        @max_requeues = max_requeues
+        @global_max_requeues = (tests.size * requeue_tolerance).ceil
       end
 
       def to_a
@@ -27,6 +29,32 @@ module CI
       def empty?
         @queue.empty?
       end
+
+      def acknowledge(test, success)
+        if !success && should_requeue?(test)
+          requeue(test)
+          return false
+        end
+        
+        true
+      end
+
+      private
+
+      attr_reader :max_requeues, :global_max_requeues
+
+      def should_requeue?(test)
+        requeues[test] < max_requeues && requeues.values.inject(0, :+) < global_max_requeues
+      end
+
+      def requeue(test)
+        requeues[test] += 1
+        @queue.unshift(test)
+      end
+
+      def requeues
+        @requeues ||= Hash.new(0)
+      end
     end
   end
 end

data/lib/ci/queue/version.rb

@@ -1,5 +1,5 @@
 module CI
   module Queue
-    VERSION = '0.1.0'
+    VERSION = '0.2.0'
   end
 end

data/lib/minitest/queue.rb

@@ -1,11 +1,77 @@
 require 'minitest'
 
+gem 'minitest-reporters', '~> 1.1'
+require 'minitest/reporters'
+
 module Minitest
+  class Requeue < Skip
+    attr_reader :failure
+
+    def initialize(failure)
+      super()
+      @failure = failure
+    end
+
+    def result_label
+      "Requeued"
+    end
+
+    def backtrace
+      failure.backtrace
+    end
+
+    def error
+      failure.error
+    end
+
+    def message
+      failure.message
+    end
+  end
+
+  module Requeueing
+    # Make requeues acts as skips for reporters not aware of the difference.
+    def skipped?
+      super || requeued?
+    end
+
+    def requeued?
+      Requeue === failure
+    end
+
+    def requeue!
+      self.failures.unshift(Requeue.new(self.failures.shift))
+    end
+  end
+
   module Queue
-    attr_accessor :queue
+    attr_reader :queue
+
+    def queue=(queue)
+      @queue = queue
+      if queue.respond_to?(:minitest_reporters)
+        self.queue_reporters = queue.minitest_reporters
+      else
+        self.queue_reporters = []
+      end
+    end
+
+    def queue_reporters=(reporters)
+      @queue_reporters ||= []
+      Reporters.reporters = ((Reporters.reporters || []) - @queue_reporters) + reporters
+      @queue_reporters = reporters
+    end
 
     SuiteNotFound = Class.new(StandardError)
 
+    def loaded_tests
+      MiniTest::Test.runnables.flat_map do |suite|
+        suite.runnable_methods.map do |method|
+          "#{suite}##{method}"
+        end
+      end
+    end
+
     def __run(*args)
       if queue
         run_from_queue(*args)
@@ -17,11 +83,15 @@ module Minitest
     def run_from_queue(reporter, *)
       runnable_classes = Minitest::Runnable.runnables.map { |s| [s.name, s] }.to_h
 
-      queue.poll do |msg|
-        class_name, method = msg.split("#".freeze, 2)
+      queue.poll do |test_name|
+        class_name, method_name = test_name.split("#".freeze, 2)
 
-        if suite = runnable_classes[class_name]
-          Minitest::Runnable.run_one_method(suite, method, reporter)
+        if klass = runnable_classes[class_name]
+          result = Minitest.run_one_method(klass, method_name)
+          unless queue.acknowledge(test_name, result.passed? || result.skipped?)
+            result.requeue!
+          end
+          reporter.record(result)
         else
           raise SuiteNotFound, "Couldn't find suite matching: #{msg.inspect}"
         end
@@ -31,3 +101,4 @@ module Minitest
 end
 
 MiniTest.singleton_class.prepend(MiniTest::Queue)
+MiniTest::Test.prepend(MiniTest::Requeueing)

data/lib/minitest/reporters/failure_formatter.rb

@@ -0,0 +1,65 @@
+require 'ansi'
+require 'delegate'
+
+module Minitest
+  module Reporters
+    class FailureFormatter < SimpleDelegator
+      include ANSI::Code
+
+      def initialize(test)
+        @test = test
+        super
+      end
+
+      def to_s
+        [
+          header,
+          body,
+          "\n"
+        ].flatten.compact.join("\n")
+      end
+
+      def to_h
+        {
+          test_and_module_name: "#{test.class}##{test.name}",
+          test_name: test.name,
+          output: to_s,
+        }
+      end
+
+      private
+
+      attr_reader :test
+
+      def header
+        "#{red(status)} #{test.class}##{test.name}"
+      end
+
+      def status
+        if test.error?
+          'ERROR'
+        elsif test.failure
+          'FAIL'
+        else
+          raise ArgumentError, "Couldn't infer test status"
+        end
+      end
+
+      def body
+        error = test.failure
+        message = if error.is_a?(MiniTest::UnexpectedError)
+          "#{error.exception.class}: #{error.exception.message}"
+        else
+          error.exception.message
+        end
+
+        backtrace = Minitest.filter_backtrace(error.backtrace).map { |line| '    ' + relativize(line) }
+        [yellow(message), *backtrace].join("\n")
+      end
+
+      def relativize(trace_line)
+        trace_line.sub(/\A#{Regexp.escape("#{Dir.pwd}/")}/, '')
+      end
+    end
+  end
+end

data/lib/minitest/reporters/queue_reporter.rb

@@ -0,0 +1,48 @@
+require 'minitest/reporters'
+
+module Minitest
+  module Reporters
+    class QueueReporter < BaseReporter
+      include ANSI::Code
+      attr_accessor :requeues
+
+      def initialize(*)
+        self.requeues = 0
+        super
+      end
+
+      def report
+        self.requeues = results.count(&:requeued?)
+        super
+        print_report
+      end
+
+      private
+
+      def print_report
+        success = failures.zero? && errors.zero?
+        failures_count = "#{failures} failures, #{errors} errors,"
+        puts [
+          'Ran %d tests, %d assertions,' % [count, assertions],
+          success ? green(failures_count) : red(failures_count),
+          yellow("#{skips} skips, #{requeues} requeues"),
+          'in %.2fs' % total_time,
+        ].join(' ')
+      end
+
+      def message_for(test)
+        e = test.failure
+
+        if test.requeued?
+          "Requeued:\n#{test.class}##{test.name} [#{location(e)}]:\n#{e.message}"
+        else
+          super
+        end
+      end
+
+      def result_line
+        "#{super}, #{requeues} requeues"
+      end
+    end
+  end
+end

data/lib/minitest/reporters/redis_reporter.rb

@@ -0,0 +1,202 @@
+require 'minitest/reporters'
+require 'minitest/reporters/failure_formatter'
+
+module Minitest
+  module Reporters
+    module RedisReporter
+      include ANSI::Code
+
+      COUNTERS = %w(
+        assertions
+        errors
+        failures
+        skips
+        requeues
+        total_time
+      ).freeze
+
+      class << self
+        attr_accessor :failure_formatter
+      end
+      self.failure_formatter = FailureFormatter
+
+      class Error
+        class << self
+          def load(payload)
+            Marshal.load(payload)
+          end
+        end
+
+        def initialize(data)
+          @data = data
+        end
+
+        def dump
+          Marshal.dump(self)
+        end
+
+        def test_name
+          @data[:test_name]
+        end
+
+        def test_and_module_name
+          @data[:test_and_module_name]
+        end
+
+        def to_s
+          output
+        end
+
+        def output
+          @data[:output]
+        end
+      end
+
+      class Base < BaseReporter
+        def initialize(redis:, build_id:, **options)
+          @redis = redis
+          @key = "build:#{build_id}"
+          super(options)
+        end
+
+        def total
+          redis.get(key('total')).to_i
+        end
+
+        def processed
+          redis.get(key('processed')).to_i
+        end
+
+        def completed?
+          total > 0 && total == processed
+        end
+
+        def error_reports
+          redis.hgetall(key('error-reports')).sort_by(&:first).map { |k, v| Error.load(v) }
+        end
+
+        private
+
+        attr_reader :redis
+
+        def key(*args)
+          [@key, *args].join(':')
+        end
+      end
+
+      class Summary < Base
+        include ANSI::Code
+
+        def report(io: STDOUT)
+          io.puts aggregates
+          errors = error_reports
+          io.puts errors
+
+          errors.empty?
+        end
+
+        def record(*)
+          raise NotImplementedError
+        end
+
+        def failures
+          fetch_summary['failures'].to_i
+        end
+
+        def errors
+          fetch_summary['errors'].to_i
+        end
+
+        def assertions
+          fetch_summary['assertions'].to_i
+        end
+
+        def skips
+          fetch_summary['skips'].to_i
+        end
+
+        def requeues
+          fetch_summary['requeues'].to_i
+        end
+
+        def total_time
+          fetch_summary['total_time'].to_f
+        end
+
+        private
+
+        def aggregates
+          success = failures.zero? && errors.zero?
+          failures_count = "#{failures} failures, #{errors} errors,"
+
+          [
+            'Ran %d tests, %d assertions,' % [processed, assertions],
+            success ? green(failures_count) : red(failures_count),
+            yellow("#{skips} skips, #{requeues} requeues"),
+            'in %.2fs (aggregated)' % total_time,
+          ].join(' ')
+        end
+
+        def fetch_summary
+          @summary ||= begin
+            counts = redis.pipelined do
+              COUNTERS.each { |c| redis.hgetall(key(c)) }
+            end
+            COUNTERS.zip(counts.map { |h| h.values.map(&:to_f).inject(:+).to_f }).to_h
+          end
+        end
+      end
+
+      class Worker < Base
+        attr_accessor :requeues
+
+        def initialize(worker_id:, **options)
+          super
+          @worker_id = worker_id
+          self.failures = 0
+          self.errors = 0
+          self.skips = 0
+          self.requeues = 0
+        end
+
+        def report
+          # noop
+        end
+
+        def record(test)
+          super
+
+          self.total_time = Minitest.clock_time - start_time
+          if test.requeued?
+            self.requeues += 1
+          elsif test.skipped?
+            self.skips += 1
+          elsif test.error?
+            self.errors += 1
+          elsif test.failure
+            self.failures += 1
+          end
+
+          redis.multi do
+            if (test.failure || test.error?) && !test.skipped?
+              redis.hset(key('error-reports'), "#{test.class}##{test.name}", dump(test))
+            else
+              redis.hdel(key('error-reports'), "#{test.class}##{test.name}")
+            end
+            COUNTERS.each do |counter|
+              redis.hset(key(counter), worker_id, send(counter))
+            end
+          end
+        end
+
+        private
+
+        def dump(test)
+          Error.new(RedisReporter.failure_formatter.new(test).to_h).dump
+        end
+
+        attr_reader :worker_id, :aggregates
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-15 00:00:00.000000000 Z
+date: 2016-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 description: To parallelize your CI without having to balance your tests
 email:
 - jean.boussier@shopify.com
@@ -102,11 +116,15 @@ files:
 - lib/ci/queue/file.rb
 - lib/ci/queue/redis.rb
 - lib/ci/queue/redis/base.rb
+- lib/ci/queue/redis/retry.rb
 - lib/ci/queue/redis/supervisor.rb
 - lib/ci/queue/redis/worker.rb
 - lib/ci/queue/static.rb
 - lib/ci/queue/version.rb
 - lib/minitest/queue.rb
+- lib/minitest/reporters/failure_formatter.rb
+- lib/minitest/reporters/queue_reporter.rb
+- lib/minitest/reporters/redis_reporter.rb
 homepage: https://github.com/Shopify/ci-queue
 licenses:
 - MIT
@@ -127,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue