ci-queue 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 11fda6e3d6ac119631a90d63086ebb9e06d628bd
+  data.tar.gz: 9ecad3efc2ec99b01e5a58f9defc55b772ffa65b
+SHA512:
+  metadata.gz: 121adc5f59a3c95c62121e6bad8ebeb321890be2a67e3f3ae14ae5b6fd5febab786b8d4a4bd4e74e54e14bc29968dc07cc3a2c54d30c7230e83a1b52cfdfb58f
+  data.tar.gz: 658a17f26818a69bb7a60c9e563a5b2e16dc4dded98637455e49313d438b3d72fa323b1a10c0179d223c8f32b95009238cddc6e145c7db46754a7114ec596ec8

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.13.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ci-queue.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Shopify
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,45 @@
+# CI::Queue
+
+Distribute tests over many workers using a queue. 
+
+## Why a queue?
+
+One big problem with distributed test suites, is test imbalance. Meaning that one worker would spend 10 minutes while all the others are done after 1 minute.
+There is algorithms available to balance perfectly your workers, but in practice your test performance tend to vary, and it's easier to consider tests as work unit in a queue and let workers pop them as fast as possible.
+
+Another advantage is that if you lose workers along the way, using a queue the other workers can pick up the job, making you resilient to failures.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ci-queue'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ci-queue
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Shopify/ci-queue.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/bundler

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# This file was generated by Bundler.
+#
+# The application 'bundler' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("bundler", "bundler")

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ci/queue"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/rake

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ci-queue.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ci/queue/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'ci-queue'
+  spec.version       = CI::Queue::VERSION
+  spec.authors       = ["Jean Boussier"]
+  spec.email         = ["jean.boussier@shopify.com"]
+
+  spec.summary       = %q{Distribute tests over many workers using a queue}
+  spec.description   = %q{To parallelize your CI without having to balance your tests}
+  spec.homepage      = 'https://github.com/Shopify/ci-queue'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', "~> 10.0"
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'redis', '~> 3.3'
+  spec.add_development_dependency 'simplecov', '~> 0.12'
+end

data/lib/ci/queue.rb

@@ -0,0 +1,3 @@
+require 'ci/queue/version'
+require 'ci/queue/static'
+require 'ci/queue/file'

data/lib/ci/queue/file.rb

@@ -0,0 +1,11 @@
+require 'ci/queue/static'
+
+module CI
+  module Queue
+    class File < Static
+      def initialize(path)
+        super(::File.readlines(path).map(&:strip).reject(&:empty?))
+      end
+    end
+  end
+end

data/lib/ci/queue/redis.rb

@@ -0,0 +1,17 @@
+require 'redis'
+require 'ci/queue/redis/base'
+require 'ci/queue/redis/worker'
+require 'ci/queue/redis/supervisor'
+
+module CI
+  module Queue
+    module Redis
+      Error = Class.new(StandardError)
+      LostMaster = Class.new(Error)
+
+      def self.new(*args)
+        Worker.new(*args)
+      end
+    end
+  end
+end

data/lib/ci/queue/redis/base.rb

@@ -0,0 +1,65 @@
+module CI
+  module Queue
+    module Redis
+      class Base
+        def initialize(redis:, build_id:)
+          @redis = redis
+          @key = "build:#{build_id}"
+        end
+
+        def empty?
+          size == 0
+        end
+
+        def size
+          redis.multi do
+            redis.llen(key('queue'))
+            redis.zcard(key('running'))
+          end.inject(:+)
+        end
+
+        def to_a
+          redis.multi do
+            redis.lrange(key('queue'), 0, -1)
+            redis.zrange(key('running'), 0, -1)
+          end.flatten.reverse
+        end
+
+        def progress
+          total - size
+        end
+
+        def wait_for_master(timeout: 10)
+          return true if master?
+          (timeout * 10 + 1).to_i.times do
+            case master_status
+            when 'ready', 'finished'
+              return true
+            else
+              sleep 0.1
+            end
+          end
+          raise LostMaster, "The master worker is still `#{master_status}` after 10 seconds waiting."
+        end
+
+        private
+
+        attr_reader :redis
+
+        def key(*args)
+          [@key, *args].join(':')
+        end
+
+        def master_status
+          redis.get(key('master-status'))
+        end
+
+        def eval_script(script, *args)
+          @scripts_cache ||= {}
+          sha = (@scripts_cache[script] ||= redis.script(:load, script))
+          redis.evalsha(sha, *args)
+        end
+      end
+    end
+  end
+end

data/lib/ci/queue/redis/supervisor.rb

@@ -0,0 +1,18 @@
+module CI
+  module Queue
+    module Redis
+      class Supervisor < Base
+        def master?
+          false
+        end
+
+        def wait_for_workers
+          return false unless wait_for_master
+
+          sleep 0.1 until empty?
+          true
+        end
+      end
+    end
+  end
+end

data/lib/ci/queue/redis/worker.rb

@@ -0,0 +1,116 @@
+module CI
+  module Queue
+    module Redis
+      class Worker < Base
+        attr_reader :total
+
+        def initialize(tests, redis:, build_id:, worker_id:, timeout:)
+          @shutdown_required = false
+          super(redis: redis, build_id: build_id)
+          @worker_id = worker_id
+          @timeout = timeout
+          push(tests)
+        end
+
+        def shutdown!
+          @shutdown_required = true
+        end
+
+        def shutdown_required?
+          @shutdown_required
+        end
+
+        def master?
+          @master
+        end
+
+        def poll
+          wait_for_master
+          while test = reserve
+            yield test
+            acknowledge(test)
+          end
+        end
+
+        def retry_queue
+          Static.new(redis.lrange(key("worker:#{worker_id}:queue"), 0, -1).reverse)
+        end
+
+        private
+
+        attr_reader :worker_id, :timeout
+
+        RESERVE_TEST = %{
+          local queue_key = KEYS[1]
+          local zset_key = KEYS[2]
+          local current_time = ARGV[1]
+
+          local test = redis.call('rpop', queue_key)
+          if test then
+            redis.call('zadd', zset_key, current_time, test)
+            return test
+          else
+            return nil
+          end
+        }
+        def reserve
+          return if shutdown_required?
+
+          if test = eval_script(RESERVE_TEST, keys: [key('queue'), key('running')], argv: [Time.now.to_f])
+            return test
+          else
+            reserve_lost_test
+          end
+        end
+
+        RESERVE_LOST_TEST = %{
+          local zset_key = KEYS[1]
+          local current_time = ARGV[1]
+          local timeout = ARGV[2]
+
+          local test = redis.call('zrangebyscore', zset_key, 0, current_time - timeout)[1]
+          if test then
+            redis.call('zadd', zset_key, current_time, test)
+            return test
+          else
+            return nil
+          end
+        }
+        def reserve_lost_test
+          until redis.zcard(key('running')) == 0
+            if test = eval_script(RESERVE_LOST_TEST, keys: [key('running')], argv: [Time.now.to_f, timeout])
+              return test
+            end
+            sleep 0.1
+          end
+          nil
+        end
+
+        ACKNOWLEDGE = %{
+          local zset_key = KEYS[1]
+          local processed_count_key = KEYS[2]
+          local test = ARGV[1]
+
+          if redis.call('zrem', zset_key, test) == 1 then
+            redis.call('incr', processed_count_key)
+          end
+        }
+        def acknowledge(test)
+          eval_script(ACKNOWLEDGE, keys: [key('running'), key('processed')], argv: [test])
+          redis.lpush(key("worker:#{worker_id}:queue"), test)
+        end
+
+        def push(tests)
+          @total = tests.size
+          if @master = redis.setnx(key('master-status'), 'setup')
+            redis.multi do
+              redis.lpush(key('queue'), tests)
+              redis.set(key('total'), @total)
+              redis.set(key('master-status'), 'ready')
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ci/queue/static.rb

@@ -0,0 +1,32 @@
+module CI
+  module Queue
+    class Static
+      attr_reader :progress, :total
+
+      def initialize(tests)
+        @queue = tests
+        @progress = 0
+        @total = tests.size
+      end
+
+      def to_a
+        @queue.dup
+      end
+
+      def size
+        @queue.size
+      end
+
+      def poll
+        while test = @queue.shift
+          yield test
+          @progress += 1
+        end
+      end
+
+      def empty?
+        @queue.empty?
+      end
+    end
+  end
+end

data/lib/ci/queue/version.rb

@@ -0,0 +1,5 @@
+module CI
+  module Queue
+    VERSION = '0.1.0'
+  end
+end

data/lib/minitest/queue.rb

@@ -0,0 +1,33 @@
+require 'minitest'
+
+module Minitest
+  module Queue
+    attr_accessor :queue
+
+    SuiteNotFound = Class.new(StandardError)
+
+    def __run(*args)
+      if queue
+        run_from_queue(*args)
+      else
+        super
+      end
+    end
+
+    def run_from_queue(reporter, *)
+      runnable_classes = Minitest::Runnable.runnables.map { |s| [s.name, s] }.to_h
+
+      queue.poll do |msg|
+        class_name, method = msg.split("#".freeze, 2)
+
+        if suite = runnable_classes[class_name]
+          Minitest::Runnable.run_one_method(suite, method, reporter)
+        else
+          raise SuiteNotFound, "Couldn't find suite matching: #{msg.inspect}"
+        end
+      end
+    end
+  end
+end
+
+MiniTest.singleton_class.prepend(MiniTest::Queue)

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: ci-queue
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jean Boussier
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-11-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+description: To parallelize your CI without having to balance your tests
+email:
+- jean.boussier@shopify.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/bundler
+- bin/console
+- bin/rake
+- bin/setup
+- ci-queue.gemspec
+- lib/ci/queue.rb
+- lib/ci/queue/file.rb
+- lib/ci/queue/redis.rb
+- lib/ci/queue/redis/base.rb
+- lib/ci/queue/redis/supervisor.rb
+- lib/ci/queue/redis/worker.rb
+- lib/ci/queue/static.rb
+- lib/ci/queue/version.rb
+- lib/minitest/queue.rb
+homepage: https://github.com/Shopify/ci-queue
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.5
+signing_key: 
+specification_version: 4
+summary: Distribute tests over many workers using a queue
+test_files: []