ci-queue 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 442794179d88fc5fc9a2c3a781b5056ff27ee013
-  data.tar.gz: 03fa2c61961e02864f122854c9c21e09c0e2b387
+  metadata.gz: 4d636ab99ab50bd22a5fc67415187e75bac147b8
+  data.tar.gz: 6825eca869948ed64897fb6e2b3ce46fc5e1a918
 SHA512:
-  metadata.gz: ca06529e6a43ec5100ce6d4da2c6ab1bd1f439f36706349e41da7b4653cd3879dc58243626687505a9fb63dd8fb714fc83c886c032c2db6c923fb24aa35c0a16
-  data.tar.gz: 7971b3b1d661227e89f1b2b807c8a060ae951748afa1ddd0eb5357285f2da39c16dbb1dd294661da605362d0dcff804f9267505cf9503e57ea35a8882eca8ff8
+  metadata.gz: 129ec21f70ad9ee2ad108537bfc7ec0032b4bbf9738973da28376a558f37dbc5de884bd4d5e9049ecf94f2ee5c02760f9a678fae67d40a92b7d2c31919f2c123
+  data.tar.gz: 5759824ed9f3b81060b926de0093d523c229f13bd98e5cefb6d4770f97da43acedd480cd97c2af92b4af5c51ecf4474a51a3461007760aa16a5de90529ec8d6c

data/.gitignore

@@ -8,3 +8,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+/lib/ci/queue/redis/*.lua
+

data/Rakefile

@@ -1,5 +1,6 @@
 require 'bundler/gem_tasks'
 require 'rake/testtask'
+require 'ci/queue/version'
 
 Rake::TestTask.new(:test) do |t|
   t.libs << 'test'
@@ -8,3 +9,18 @@ Rake::TestTask.new(:test) do |t|
 end
 
 task :default => :test
+
+namespace :scripts do
+  task :copy do
+    Dir[File.join(CI::Queue::DEV_SCRIPTS_ROOT, '*.lua')].each do |origin|
+      filename = File.basename(origin)
+      destination = File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, filename)
+      File.open(destination, 'w+') do |f|
+        f.write("-- AUTOGENERATED FILE DO NOT EDIT DIRECTLY\n")
+        f.write(File.read(origin))
+      end
+    end
+  end
+end
+
+Rake::Task['build'].enhance ['scripts:copy']

data/ci-queue.gemspec

@@ -2,6 +2,10 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'ci/queue/version'
+require 'pathname'
+
+dir = Pathname.new(CI::Queue::RELEASE_SCRIPTS_ROOT).relative_path_from(Pathname.new(__dir__).realpath)
+lua_scripts = Dir[dir.join('*.lua').to_s]
 
 Gem::Specification.new do |spec|
   spec.name          = 'ci-queue'
@@ -14,9 +18,10 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/Shopify/ci-queue'
   spec.license       = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.files         = lua_scripts + `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
+
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']

data/lib/ci/queue/redis/base.rb

@@ -64,7 +64,13 @@ module CI
 
         def load_script(script)
           @scripts_cache ||= {}
-          @scripts_cache[script] ||= redis.script(:load, script)
+          @scripts_cache[script] ||= redis.script(:load, read_script(script))
+        end
+
+        def read_script(name)
+          ::File.read(::File.join(CI::Queue::DEV_SCRIPTS_ROOT, "#{name}.lua"))
+        rescue SystemCallError
+          ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
         end
       end
     end

data/lib/ci/queue/redis/worker.rb

@@ -71,53 +71,18 @@ module CI
 
         def acknowledge(test)
           raise_on_mismatching_test(test)
-          ack(test)
+          eval_script(
+            :acknowledge,
+            keys: [key('running'), key('processed')],
+            argv: [test],
+          ) == 1
         end
 
-        REQUEUE = %{
-          local processed_key = KEYS[1]
-          local requeues_count_key = KEYS[2]
-          local queue_key = KEYS[3]
-          local zset_key = KEYS[4]
-
-          local max_requeues = tonumber(ARGV[1])
-          local global_max_requeues = tonumber(ARGV[2])
-          local test = ARGV[3]
-          local offset = ARGV[4]
-
-          if redis.call('sismember', processed_key, test) == 1 then
-            return false
-          end
-
-          local global_requeues = tonumber(redis.call('hget', requeues_count_key, '___total___'))
-          if global_requeues and global_requeues >= tonumber(global_max_requeues) then
-            return false
-          end
-
-          local requeues = tonumber(redis.call('hget', requeues_count_key, test))
-          if requeues and requeues >= max_requeues then
-            return false
-          end
-
-          redis.call('hincrby', requeues_count_key, '___total___', 1)
-          redis.call('hincrby', requeues_count_key, test, 1)
-
-          local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
-          if pivot then
-            redis.call('linsert', queue_key, 'BEFORE', pivot, test)
-          else
-            redis.call('lpush', queue_key, test)
-          end
-
-          redis.call('zrem', zset_key, test)
-
-          return true
-        }
         def requeue(test, offset: Redis.requeue_offset)
           raise_on_mismatching_test(test)
 
           requeued = eval_script(
-            REQUEUE,
+            :requeue,
             keys: [key('processed'), key('requeues-count'), key('queue'), key('running')],
             argv: [max_requeues, global_max_requeues, test, offset],
           ) == 1
@@ -148,71 +113,24 @@ module CI
         end
 
         RESERVE_TEST = %{
-          local queue_key = KEYS[1]
-          local zset_key = KEYS[2]
-          local processed_key = KEYS[3]
-
-          local current_time = ARGV[1]
-
-          local test = redis.call('rpop', queue_key)
-          if test then
-            redis.call('zadd', zset_key, current_time, test)
-            return test
-          else
-            return nil
-          end
         }
+
         def try_to_reserve_test
           eval_script(
-            RESERVE_TEST,
-            keys: [key('queue'), key('running'), key('processed')],
+            :reserve,
+            keys: [key('queue'), key('running'), key('processed'), key('worker', worker_id, 'queue')],
             argv: [Time.now.to_f],
           )
         end
 
-        RESERVE_LOST_TEST = %{
-          local zset_key = KEYS[1]
-          local processed_key = KEYS[2]
-          local current_time = ARGV[1]
-          local timeout = ARGV[2]
-
-          local lost_tests = redis.call('zrangebyscore', zset_key, 0, current_time - timeout)
-          for _, test in ipairs(lost_tests) do
-            if redis.call('sismember', processed_key, test) == 0 then
-              redis.call('zadd', zset_key, current_time, test)
-              return test
-            end
-          end
-
-          return nil
-        }
         def try_to_reserve_lost_test
           eval_script(
-            RESERVE_LOST_TEST,
-            keys: [key('running'), key('completed')],
+            :reserve_lost,
+            keys: [key('running'), key('completed'), key('worker', worker_id, 'queue')],
             argv: [Time.now.to_f, timeout],
           )
         end
 
-        ACKNOWLEDGE = %{
-          local zset_key = KEYS[1]
-          local processed_key = KEYS[2]
-
-          local worker_id = ARGV[1]
-          local test = ARGV[2]
-
-          redis.call('zrem', zset_key, test)
-          return redis.call('sadd', processed_key, test)
-        }
-        def ack(test)
-          redis.lpush(key('worker', worker_id, 'queue'), test)
-          eval_script(
-            ACKNOWLEDGE,
-            keys: [key('running'), key('processed')],
-            argv: [worker_id, test],
-          ) == 1
-        end
-
         def push(tests)
           @total = tests.size
           if @master = redis.setnx(key('master-status'), 'setup')

data/lib/ci/queue/version.rb

@@ -1,5 +1,7 @@
 module CI
   module Queue
-    VERSION = '0.5.2'
+    VERSION = '0.6.0'
+    DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
+    RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end
 end

data/lib/minitest/reporters/redis_reporter.rb

@@ -22,17 +22,21 @@ module Minitest
 
       class Error
         class << self
+          attr_accessor :coder
+
           def load(payload)
-            Marshal.load(payload)
+            new(coder.load(payload))
           end
         end
 
+        self.coder = Marshal
+
         def initialize(data)
           @data = data
         end
 
         def dump
-          Marshal.dump(self)
+          self.class.coder.dump(@data)
         end
 
         def test_name

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-13 00:00:00.000000000 Z
+date: 2017-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -102,10 +102,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".travis.yml"
 - Gemfile
-- LICENSE.txt
-- README.md
 - Rakefile
 - bin/bundler
 - bin/console
@@ -116,7 +113,11 @@ files:
 - lib/ci/queue.rb
 - lib/ci/queue/file.rb
 - lib/ci/queue/redis.rb
+- lib/ci/queue/redis/acknowledge.lua
 - lib/ci/queue/redis/base.rb
+- lib/ci/queue/redis/requeue.lua
+- lib/ci/queue/redis/reserve.lua
+- lib/ci/queue/redis/reserve_lost.lua
 - lib/ci/queue/redis/retry.rb
 - lib/ci/queue/redis/supervisor.rb
 - lib/ci/queue/redis/worker.rb
@@ -147,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue

data/.travis.yml

@@ -1,7 +0,0 @@
-sudo: false
-services:
-  - redis-server
-language: ruby
-rvm:
-  - 2.3.1
-before_install: gem install bundler -v 1.13.6

data/LICENSE.txt

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2016 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

data/README.md

@@ -1,48 +0,0 @@
-# CI::Queue
-
-[![Gem Version](https://badge.fury.io/rb/ci-queue.svg)](https://rubygems.org/gems/ci-queue)
-[![Build Status](https://travis-ci.org/Shopify/ci-queue.svg?branch=master)](https://travis-ci.org/Shopify/ci-queue)
-
-Distribute tests over many workers using a queue. 
-
-## Why a queue?
-
-One big problem with distributed test suites, is test imbalance. Meaning that one worker would spend 10 minutes while all the others are done after 1 minute.
-There is algorithms available to balance perfectly your workers, but in practice your test performance tend to vary, and it's easier to consider tests as work unit in a queue and let workers pop them as fast as possible.
-
-Another advantage is that if you lose workers along the way, using a queue the other workers can pick up the job, making you resilient to failures.
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'ci-queue'
-```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install ci-queue
-
-## Usage
-
-TODO: Write usage instructions here
-
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
-## Contributing
-
-Bug reports and pull requests are welcome on GitHub at https://github.com/Shopify/ci-queue.
-
-## License
-
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
-