ci-queue 0.15.1 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be97720fd013ba146232d7ed706bbfde45525263e84ecb99e780b74e526b8c77
-  data.tar.gz: 6868d32c98c70fbbeb3583ec4f613b117288370c91e900766fd7e3e1b9aa6426
+  metadata.gz: bce03b836af2ca47628929525648bed8d667a434b3d09228d33e0429cbb6a3a4
+  data.tar.gz: 99e6734a053804f6104409822ac2292c26bcd373d060f34611a1e0fd20be3aa3
 SHA512:
-  metadata.gz: 85c8b1febf77dd9971637dd54b9bef89d966fb934dc2ec639cd41c33cd6ec11b82e478f136e802ac9e6b8653d4e57b72abf75f87f7fe0814683222828d60db65
-  data.tar.gz: 2fd24846567dc7e1569ee932e7e0674ab893c67061fd1007cf57366efb4220309dc62405587dd2d701c2366b864fc32144024cf8ef5312245c00dc45a1d70af0
+  metadata.gz: 3c8cf25173e9282dc101252ca6f494b94dcbc5c7baa6dd9283320aae47e9413a6139f78956ba999ed24aa4a876278f0f45fcede73a8b821039960ce4d8326e22
+  data.tar.gz: e96bbc23377e38e7bcd36239adb8e5b05c4d8488601972f744f5484d1b569e5a35de85fb56de6b89f912e525728ed3200c08c78c0c81db8ed932e79e86338c77

data/lib/ci/queue.rb

@@ -17,6 +17,10 @@ module CI
 
     attr_accessor :shuffler
 
+    module Warnings
+      RESERVED_LOST_TEST = :RESERVED_LOST_TEST
+    end
+
     def shuffle(tests, random)
       if shuffler
         shuffler.call(tests, random)

data/lib/ci/queue/redis/build_record.rb

@@ -20,6 +20,19 @@ module CI
           redis.hkeys(key('error-reports'))
         end
 
+        def pop_warnings
+          warnings = redis.multi do
+            redis.lrange(key('warnings'), 0, -1)
+            redis.del(key('warnings'))
+          end.first
+
+          warnings.map { |p| Marshal.load(p) }
+        end
+
+        def record_warning(type, attributes)
+          redis.rpush(key('warnings'), Marshal.dump([type, attributes]))
+        end
+
         def record_error(id, payload, stats: nil)
           redis.pipelined do
             redis.hset(

data/lib/ci/queue/redis/supervisor.rb

@@ -18,10 +18,14 @@ module CI
         def wait_for_workers
           return false unless wait_for_master(timeout: config.timeout)
 
+          yield if block_given?
+
           time_left = config.timeout
           until exhausted? || time_left <= 0
-            sleep 0.1
-            time_left -= 0.1
+            sleep 1
+            time_left -= 1
+
+            yield if block_given?
           end
           exhausted?
         rescue CI::Queue::Redis::LostMaster

data/lib/ci/queue/redis/worker.rb

@@ -46,7 +46,7 @@ module CI
           wait_for_master
           until shutdown_required? || config.circuit_breaker.open? || exhausted?
             if test = reserve
-              yield index.fetch(test)
+              yield index.fetch(test), @last_warning
             else
               sleep 0.05
             end
@@ -140,11 +140,17 @@ module CI
         end
 
         def try_to_reserve_lost_test
-          eval_script(
+          lost_test = eval_script(
             :reserve_lost,
             keys: [key('running'), key('completed'), key('worker', worker_id, 'queue')],
             argv: [Time.now.to_f, timeout],
           )
+
+          if lost_test
+            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: timeout)
+          end
+
+          lost_test
         end
 
         def push(tests)

data/lib/ci/queue/version.rb

@@ -1,6 +1,6 @@
 module CI
   module Queue
-    VERSION = '0.15.1'
+    VERSION = '0.16.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -133,7 +133,7 @@ module Minitest
 
         step("Waiting for workers to complete")
 
-        unless supervisor.wait_for_workers
+        unless supervisor.wait_for_workers { display_warnings(supervisor.build) }
           unless supervisor.queue_initialized?
             abort! "No master was elected. Did all workers crash?"
           end
@@ -153,6 +153,20 @@ module Minitest
       attr_reader :queue_config, :options, :command, :argv
       attr_accessor :queue, :queue_url, :load_paths
 
+      def display_warnings(build)
+        build.pop_warnings.each do |type, attributes|
+          case type
+          when CI::Queue::Warnings::RESERVED_LOST_TEST
+            puts reopen_previous_step
+            puts yellow(
+              "[WARNING] #{attributes[:test]} was picked up by another worker because it didn't complete in the allocated #{attributes[:timeout]} seconds.\n" \
+              "You may want to either optimize this test of bump ci-queue timeout.\n" \
+              "It's also possible that the worker that was processing it was terminated without being able to report back.\n"
+            )
+          end
+        end
+      end
+
       def run_tests_in_fork(queue)
         child_pid = fork do
           Minitest.queue = queue

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.16.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-05 00:00:00.000000000 Z
+date: 2019-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi