ci-queue 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75fc560a4dbae8a7ad7487570afaef29048788b8
-  data.tar.gz: 1f84cff31f94272e01e436e70a3f4a66a9dc70d5
+  metadata.gz: c56cc78cb306fb67c398a2cdf41b5a9d7d594d87
+  data.tar.gz: d63fb0d78ec9fe9eeb04bc2bc60a1ab6aeeab77d
 SHA512:
-  metadata.gz: af05def95dc288e9a78b9fa5ccae1d62080d74a37f50ed3702cbceefc7da6ea641178dd988d1b02e1ae574bb57aa07e4fcfa7a6d4f434e79f86c23248623efe2
-  data.tar.gz: 18930ed5ca16d136100658302e23509062245259cc3e7cc2749c9c4e7812e7a28c692799cfaad85e982aec8bd34b5593c4f898d8e5851550a53f2cc738b1f32c
+  metadata.gz: 4ef344549d3bb3e75bb01f0e9bae693b2ed635eedcc31f5043383de2c8775e662c1822ba740435b174e0f9ebce3346b97ccc203a44ce433ab922ab9796d2a91f
+  data.tar.gz: 0b6ec24e92bf4ec95d6bd68a20e64efab06dcf5109bb9c7b3ab1ad2f8632cc8907865dbaacebff891606f817d0101376281ba5549d152b8acfc3ebfe537eeea9

data/ci-queue.gemspec

@@ -35,4 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'redis', '~> 3.3'
   spec.add_development_dependency 'simplecov', '~> 0.12'
   spec.add_development_dependency 'minitest-reporters', '~> 1.1'
+
+  spec.add_development_dependency 'snappy'
+  spec.add_development_dependency 'msgpack'
 end

data/lib/ci/queue.rb

@@ -3,7 +3,6 @@ require 'cgi'
 
 require 'ci/queue/version'
 require 'ci/queue/output_helpers'
-require 'ci/queue/index'
 require 'ci/queue/configuration'
 require 'ci/queue/static'
 require 'ci/queue/file'
@@ -13,6 +12,16 @@ module CI
   module Queue
     extend self
 
+    attr_accessor :shuffler
+
+    def shuffle(tests, random)
+      if shuffler
+        shuffler.call(tests, random)
+      else
+        tests.sort.shuffle(random: random)
+      end
+    end
+
     def from_uri(url, config)
       uri = URI(url)
       implementation = case uri.scheme

data/lib/ci/queue/bisect.rb

@@ -10,9 +10,8 @@ module CI
         @tests.size
       end
 
-      def populate(all_tests, &test_indexer)
+      def populate(all_tests, random: nil)
         @all_tests = all_tests
-        @test_indexer = test_indexer
       end
 
       def to_a
@@ -24,11 +23,11 @@ module CI
       end
 
       def failing_test
-        Static.new([config.failing_test], config).populate(@all_tests, &@test_indexer)
+        Static.new([config.failing_test], config).populate(@all_tests)
       end
 
       def candidates
-        Static.new(first_half + [config.failing_test], config).populate(@all_tests, &@test_indexer)
+        Static.new(first_half + [config.failing_test], config).populate(@all_tests)
       end
 
       def failed!

data/lib/ci/queue/redis/worker.rb

@@ -19,9 +19,10 @@ module CI
           super(redis, config)
         end
 
-        def populate(tests, &indexer)
-          @index = Index.new(tests, &indexer)
-          push(tests.map { |t| index.key(t) })
+        def populate(tests, random: Random.new)
+          @index = tests.map { |t| [t.id, t] }.to_h
+          tests = Queue.shuffle(tests, random)
+          push(tests.map(&:id))
           self
         end
 
@@ -76,7 +77,7 @@ module CI
         end
 
         def acknowledge(test)
-          test_key = index.key(test)
+          test_key = test.id
           raise_on_mismatching_test(test_key)
           eval_script(
             :acknowledge,
@@ -86,7 +87,7 @@ module CI
         end
 
         def requeue(test, offset: Redis.requeue_offset)
-          test_key = index.key(test)
+          test_key = test.id
           raise_on_mismatching_test(test_key)
 
           requeued = eval_script(

data/lib/ci/queue/static.rb

@@ -32,8 +32,8 @@ module CI
         self
       end
 
-      def populate(tests, &indexer)
-        @index = Index.new(tests, &indexer)
+      def populate(tests, random: nil)
+        @index = tests.map { |t| [t.id, t] }.to_h
         self
       end
 
@@ -65,10 +65,10 @@ module CI
       end
 
       def requeue(test)
-        key = index.key(test)
-        return false unless should_requeue?(key)
-        requeues[key] += 1
-        @queue.unshift(index.key(test))
+        test_key = test.id
+        return false unless should_requeue?(test_key)
+        requeues[test_key] += 1
+        @queue.unshift(test_key)
         true
       end
 

data/lib/ci/queue/version.rb

@@ -1,6 +1,6 @@
 module CI
   module Queue
-    VERSION = '0.9.1'
+    VERSION = '0.9.2'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue.rb

@@ -45,6 +45,25 @@ module Minitest
   end
 
   module Queue
+    class SingleExample
+      def initialize(runnable, method_name)
+        @runnable = runnable
+        @method_name = method_name
+      end
+
+      def id
+        @id ||= "#{@runnable}##{@method_name}"
+      end
+
+      def <=>(other)
+        id <=> other.id
+      end
+
+      def run
+        Minitest.run_one_method(@runnable, @method_name)
+      end
+    end
+
     attr_reader :queue
 
     def queue=(queue)
@@ -63,12 +82,10 @@ module Minitest
       @queue_reporters = reporters
     end
 
-    SuiteNotFound = Class.new(StandardError)
-
     def loaded_tests
-      Minitest::Test.runnables.flat_map do |suite|
-        suite.runnable_methods.map do |method|
-          "#{suite}##{method}"
+      Minitest::Test.runnables.flat_map do |runnable|
+        runnable.runnable_methods.map do |method_name|
+          SingleExample.new(runnable, method_name)
         end
       end
     end
@@ -82,24 +99,16 @@ module Minitest
     end
 
     def run_from_queue(reporter, *)
-      runnable_classes = Minitest::Runnable.runnables.map { |s| [s.name, s] }.to_h
-
-      queue.poll do |test_name|
-        class_name, method_name = test_name.split("#".freeze, 2)
-
-        if klass = runnable_classes[class_name]
-          result = Minitest.run_one_method(klass, method_name)
-          failed = !(result.passed? || result.skipped?)
-          if failed && queue.requeue(test_name)
-            result.requeue!
-            reporter.record(result)
-          elsif queue.acknowledge(test_name) || !failed
-            # If the test was already acknowledged by another worker (we timed out)
-            # Then we only record it if it is successful.
-            reporter.record(result)
-          end
-        else
-          raise SuiteNotFound, "Couldn't find suite matching: #{test_name}"
+      queue.poll do |example|
+        result = example.run
+        failed = !(result.passed? || result.skipped?)
+        if failed && queue.requeue(example)
+          result.requeue!
+          reporter.record(result)
+        elsif queue.acknowledge(example) || !failed
+          # If the test was already acknowledged by another worker (we timed out)
+          # Then we only record it if it is successful.
+          reporter.record(result)
         end
       end
     end

data/lib/minitest/queue/runner.rb

@@ -89,7 +89,7 @@ module Minitest
           command += argv
 
           puts
-          puts "cat <<EOF |\n#{failing_order.to_a.join("\n")}\nEOF\n#{command.join(' ')}"
+          puts "cat <<EOF |\n#{failing_order.to_a.map(&:id).join("\n")}\nEOF\n#{command.join(' ')}"
           puts
           exit! 0
         end
@@ -139,7 +139,7 @@ module Minitest
       end
 
       def populate_queue
-        Minitest.queue.populate(shuffle(Minitest.loaded_tests), &:to_s) # TODO: stop serializing
+        Minitest.queue.populate(Minitest.loaded_tests, random: ordering_seed, &:id)
       end
 
       def set_load_path
@@ -283,10 +283,12 @@ module Minitest
         string.lines.map(&:strip)
       end
 
-      def shuffle(tests)
-        return tests unless queue_config.seed
-        random = Random.new(Digest::MD5.hexdigest(queue_config.seed).to_i(16))
-        tests.shuffle(random: random)
+      def ordering_seed
+        if queue_config.seed
+          Random.new(Digest::MD5.hexdigest(queue_config.seed).to_i(16))
+        else
+          Random.new
+        end
       end
 
       def queue_url

data/lib/minitest/reporters/redis_reporter.rb

@@ -31,6 +31,36 @@ module Minitest
 
         self.coder = Marshal
 
+        begin
+          require 'snappy'
+          require 'msgpack'
+          require 'stringio'
+
+          module SnappyPack
+            extend self
+
+            MSGPACK = MessagePack::Factory.new
+            MSGPACK.register_type(0x00, Symbol)
+
+            def load(payload)
+              io = StringIO.new(Snappy.inflate(payload))
+              MSGPACK.unpacker(io).unpack
+            end
+
+            def dump(object)
+              io = StringIO.new
+              packer = MSGPACK.packer(io)
+              packer.pack(object)
+              packer.flush
+              io.rewind
+              Snappy.deflate(io.string).force_encoding(Encoding::UTF_8)
+            end
+          end
+
+          self.coder = SnappyPack
+        rescue LoadError
+        end
+
         def initialize(data)
           @data = data
         end

data/lib/rspec/queue.rb

@@ -126,6 +126,10 @@ module RSpec
         example.id
       end
 
+      def <=>(other)
+        id <=> other.id
+      end
+
       def run(reporter)
         return if RSpec.world.wants_to_quit
         instance = example_group.new(example.inspect_output)
@@ -156,7 +160,7 @@ module RSpec
         end
 
         queue = CI::Queue.from_uri(queue_url, RSpec::Queue.config)
-        queue.populate(shuffle(examples), &:id)
+        queue.populate(examples, random: ordering_seed, &:id)
         examples_count = examples.size # TODO: figure out which stub value would be best
         success = true
         @configuration.reporter.report(examples_count) do |reporter|
@@ -174,10 +178,12 @@ module RSpec
 
       private
 
-      def shuffle(examples)
-        return examples unless RSpec::Queue.config.seed
-        random = Random.new(Digest::MD5.hexdigest(RSpec::Queue.config.seed).to_i(16))
-        examples.shuffle(random: random)
+      def ordering_seed
+        if RSpec::Queue.config.seed
+          Random.new(Digest::MD5.hexdigest(RSpec::Queue.config.seed).to_i(16))
+        else
+          Random.new
+        end
       end
 
       def queue_url

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-30 00:00:00.000000000 Z
+date: 2017-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi
@@ -122,6 +122,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: snappy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: To parallelize your CI without having to balance your tests
 email:
 - jean.boussier@shopify.com
@@ -147,7 +175,6 @@ files:
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/configuration.rb
 - lib/ci/queue/file.rb
-- lib/ci/queue/index.rb
 - lib/ci/queue/output_helpers.rb
 - lib/ci/queue/redis.rb
 - lib/ci/queue/redis/acknowledge.lua

data/lib/ci/queue/index.rb

@@ -1,20 +0,0 @@
-module CI
-  module Queue
-    class Index
-      def initialize(objects, &indexer)
-        @index = objects.map { |o| [indexer.call(o), o] }.to_h
-        @indexer = indexer
-      end
-
-      def fetch(key)
-        @index.fetch(key)
-      end
-
-      def key(value)
-        key = @indexer.call(value)
-        raise KeyError, "value not found: #{value.inspect}" unless @index.key?(key)
-        key
-      end
-    end
-  end
-end