ci-queue 0.5.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 442794179d88fc5fc9a2c3a781b5056ff27ee013
4
- data.tar.gz: 03fa2c61961e02864f122854c9c21e09c0e2b387
3
+ metadata.gz: 4d636ab99ab50bd22a5fc67415187e75bac147b8
4
+ data.tar.gz: 6825eca869948ed64897fb6e2b3ce46fc5e1a918
5
5
  SHA512:
6
- metadata.gz: ca06529e6a43ec5100ce6d4da2c6ab1bd1f439f36706349e41da7b4653cd3879dc58243626687505a9fb63dd8fb714fc83c886c032c2db6c923fb24aa35c0a16
7
- data.tar.gz: 7971b3b1d661227e89f1b2b807c8a060ae951748afa1ddd0eb5357285f2da39c16dbb1dd294661da605362d0dcff804f9267505cf9503e57ea35a8882eca8ff8
6
+ metadata.gz: 129ec21f70ad9ee2ad108537bfc7ec0032b4bbf9738973da28376a558f37dbc5de884bd4d5e9049ecf94f2ee5c02760f9a678fae67d40a92b7d2c31919f2c123
7
+ data.tar.gz: 5759824ed9f3b81060b926de0093d523c229f13bd98e5cefb6d4770f97da43acedd480cd97c2af92b4af5c51ecf4474a51a3461007760aa16a5de90529ec8d6c
data/.gitignore CHANGED
@@ -8,3 +8,5 @@
8
8
  /pkg/
9
9
  /spec/reports/
10
10
  /tmp/
11
+ /lib/ci/queue/redis/*.lua
12
+
data/Rakefile CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'bundler/gem_tasks'
2
2
  require 'rake/testtask'
3
+ require 'ci/queue/version'
3
4
 
4
5
  Rake::TestTask.new(:test) do |t|
5
6
  t.libs << 'test'
@@ -8,3 +9,18 @@ Rake::TestTask.new(:test) do |t|
8
9
  end
9
10
 
10
11
  task :default => :test
12
+
13
+ namespace :scripts do
14
+ task :copy do
15
+ Dir[File.join(CI::Queue::DEV_SCRIPTS_ROOT, '*.lua')].each do |origin|
16
+ filename = File.basename(origin)
17
+ destination = File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, filename)
18
+ File.open(destination, 'w+') do |f|
19
+ f.write("-- AUTOGENERATED FILE DO NOT EDIT DIRECTLY\n")
20
+ f.write(File.read(origin))
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ Rake::Task['build'].enhance ['scripts:copy']
@@ -2,6 +2,10 @@
2
2
  lib = File.expand_path('../lib', __FILE__)
3
3
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
  require 'ci/queue/version'
5
+ require 'pathname'
6
+
7
+ dir = Pathname.new(CI::Queue::RELEASE_SCRIPTS_ROOT).relative_path_from(Pathname.new(__dir__).realpath)
8
+ lua_scripts = Dir[dir.join('*.lua').to_s]
5
9
 
6
10
  Gem::Specification.new do |spec|
7
11
  spec.name = 'ci-queue'
@@ -14,9 +18,10 @@ Gem::Specification.new do |spec|
14
18
  spec.homepage = 'https://github.com/Shopify/ci-queue'
15
19
  spec.license = 'MIT'
16
20
 
17
- spec.files = `git ls-files -z`.split("\x0").reject do |f|
21
+ spec.files = lua_scripts + `git ls-files -z`.split("\x0").reject do |f|
18
22
  f.match(%r{^(test|spec|features)/})
19
23
  end
24
+
20
25
  spec.bindir = 'exe'
21
26
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
22
27
  spec.require_paths = ['lib']
@@ -64,7 +64,13 @@ module CI
64
64
 
65
65
  def load_script(script)
66
66
  @scripts_cache ||= {}
67
- @scripts_cache[script] ||= redis.script(:load, script)
67
+ @scripts_cache[script] ||= redis.script(:load, read_script(script))
68
+ end
69
+
70
+ def read_script(name)
71
+ ::File.read(::File.join(CI::Queue::DEV_SCRIPTS_ROOT, "#{name}.lua"))
72
+ rescue SystemCallError
73
+ ::File.read(::File.join(CI::Queue::RELEASE_SCRIPTS_ROOT, "#{name}.lua"))
68
74
  end
69
75
  end
70
76
  end
@@ -71,53 +71,18 @@ module CI
71
71
 
72
72
  def acknowledge(test)
73
73
  raise_on_mismatching_test(test)
74
- ack(test)
74
+ eval_script(
75
+ :acknowledge,
76
+ keys: [key('running'), key('processed')],
77
+ argv: [test],
78
+ ) == 1
75
79
  end
76
80
 
77
- REQUEUE = %{
78
- local processed_key = KEYS[1]
79
- local requeues_count_key = KEYS[2]
80
- local queue_key = KEYS[3]
81
- local zset_key = KEYS[4]
82
-
83
- local max_requeues = tonumber(ARGV[1])
84
- local global_max_requeues = tonumber(ARGV[2])
85
- local test = ARGV[3]
86
- local offset = ARGV[4]
87
-
88
- if redis.call('sismember', processed_key, test) == 1 then
89
- return false
90
- end
91
-
92
- local global_requeues = tonumber(redis.call('hget', requeues_count_key, '___total___'))
93
- if global_requeues and global_requeues >= tonumber(global_max_requeues) then
94
- return false
95
- end
96
-
97
- local requeues = tonumber(redis.call('hget', requeues_count_key, test))
98
- if requeues and requeues >= max_requeues then
99
- return false
100
- end
101
-
102
- redis.call('hincrby', requeues_count_key, '___total___', 1)
103
- redis.call('hincrby', requeues_count_key, test, 1)
104
-
105
- local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
106
- if pivot then
107
- redis.call('linsert', queue_key, 'BEFORE', pivot, test)
108
- else
109
- redis.call('lpush', queue_key, test)
110
- end
111
-
112
- redis.call('zrem', zset_key, test)
113
-
114
- return true
115
- }
116
81
  def requeue(test, offset: Redis.requeue_offset)
117
82
  raise_on_mismatching_test(test)
118
83
 
119
84
  requeued = eval_script(
120
- REQUEUE,
85
+ :requeue,
121
86
  keys: [key('processed'), key('requeues-count'), key('queue'), key('running')],
122
87
  argv: [max_requeues, global_max_requeues, test, offset],
123
88
  ) == 1
@@ -148,71 +113,24 @@ module CI
148
113
  end
149
114
 
150
115
  RESERVE_TEST = %{
151
- local queue_key = KEYS[1]
152
- local zset_key = KEYS[2]
153
- local processed_key = KEYS[3]
154
-
155
- local current_time = ARGV[1]
156
-
157
- local test = redis.call('rpop', queue_key)
158
- if test then
159
- redis.call('zadd', zset_key, current_time, test)
160
- return test
161
- else
162
- return nil
163
- end
164
116
  }
117
+
165
118
  def try_to_reserve_test
166
119
  eval_script(
167
- RESERVE_TEST,
168
- keys: [key('queue'), key('running'), key('processed')],
120
+ :reserve,
121
+ keys: [key('queue'), key('running'), key('processed'), key('worker', worker_id, 'queue')],
169
122
  argv: [Time.now.to_f],
170
123
  )
171
124
  end
172
125
 
173
- RESERVE_LOST_TEST = %{
174
- local zset_key = KEYS[1]
175
- local processed_key = KEYS[2]
176
- local current_time = ARGV[1]
177
- local timeout = ARGV[2]
178
-
179
- local lost_tests = redis.call('zrangebyscore', zset_key, 0, current_time - timeout)
180
- for _, test in ipairs(lost_tests) do
181
- if redis.call('sismember', processed_key, test) == 0 then
182
- redis.call('zadd', zset_key, current_time, test)
183
- return test
184
- end
185
- end
186
-
187
- return nil
188
- }
189
126
  def try_to_reserve_lost_test
190
127
  eval_script(
191
- RESERVE_LOST_TEST,
192
- keys: [key('running'), key('completed')],
128
+ :reserve_lost,
129
+ keys: [key('running'), key('completed'), key('worker', worker_id, 'queue')],
193
130
  argv: [Time.now.to_f, timeout],
194
131
  )
195
132
  end
196
133
 
197
- ACKNOWLEDGE = %{
198
- local zset_key = KEYS[1]
199
- local processed_key = KEYS[2]
200
-
201
- local worker_id = ARGV[1]
202
- local test = ARGV[2]
203
-
204
- redis.call('zrem', zset_key, test)
205
- return redis.call('sadd', processed_key, test)
206
- }
207
- def ack(test)
208
- redis.lpush(key('worker', worker_id, 'queue'), test)
209
- eval_script(
210
- ACKNOWLEDGE,
211
- keys: [key('running'), key('processed')],
212
- argv: [worker_id, test],
213
- ) == 1
214
- end
215
-
216
134
  def push(tests)
217
135
  @total = tests.size
218
136
  if @master = redis.setnx(key('master-status'), 'setup')
@@ -1,5 +1,7 @@
1
1
  module CI
2
2
  module Queue
3
- VERSION = '0.5.2'
3
+ VERSION = '0.6.0'
4
+ DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
5
+ RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
4
6
  end
5
7
  end
@@ -22,17 +22,21 @@ module Minitest
22
22
 
23
23
  class Error
24
24
  class << self
25
+ attr_accessor :coder
26
+
25
27
  def load(payload)
26
- Marshal.load(payload)
28
+ new(coder.load(payload))
27
29
  end
28
30
  end
29
31
 
32
+ self.coder = Marshal
33
+
30
34
  def initialize(data)
31
35
  @data = data
32
36
  end
33
37
 
34
38
  def dump
35
- Marshal.dump(self)
39
+ self.class.coder.dump(@data)
36
40
  end
37
41
 
38
42
  def test_name
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ci-queue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jean Boussier
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-06-13 00:00:00.000000000 Z
11
+ date: 2017-08-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -102,10 +102,7 @@ extensions: []
102
102
  extra_rdoc_files: []
103
103
  files:
104
104
  - ".gitignore"
105
- - ".travis.yml"
106
105
  - Gemfile
107
- - LICENSE.txt
108
- - README.md
109
106
  - Rakefile
110
107
  - bin/bundler
111
108
  - bin/console
@@ -116,7 +113,11 @@ files:
116
113
  - lib/ci/queue.rb
117
114
  - lib/ci/queue/file.rb
118
115
  - lib/ci/queue/redis.rb
116
+ - lib/ci/queue/redis/acknowledge.lua
119
117
  - lib/ci/queue/redis/base.rb
118
+ - lib/ci/queue/redis/requeue.lua
119
+ - lib/ci/queue/redis/reserve.lua
120
+ - lib/ci/queue/redis/reserve_lost.lua
120
121
  - lib/ci/queue/redis/retry.rb
121
122
  - lib/ci/queue/redis/supervisor.rb
122
123
  - lib/ci/queue/redis/worker.rb
@@ -147,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
147
148
  version: '0'
148
149
  requirements: []
149
150
  rubyforge_project:
150
- rubygems_version: 2.2.3
151
+ rubygems_version: 2.6.13
151
152
  signing_key:
152
153
  specification_version: 4
153
154
  summary: Distribute tests over many workers using a queue
@@ -1,7 +0,0 @@
1
- sudo: false
2
- services:
3
- - redis-server
4
- language: ruby
5
- rvm:
6
- - 2.3.1
7
- before_install: gem install bundler -v 1.13.6
@@ -1,21 +0,0 @@
1
- The MIT License (MIT)
2
-
3
- Copyright (c) 2016 Shopify
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in
13
- all copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
- THE SOFTWARE.
data/README.md DELETED
@@ -1,48 +0,0 @@
1
- # CI::Queue
2
-
3
- [![Gem Version](https://badge.fury.io/rb/ci-queue.svg)](https://rubygems.org/gems/ci-queue)
4
- [![Build Status](https://travis-ci.org/Shopify/ci-queue.svg?branch=master)](https://travis-ci.org/Shopify/ci-queue)
5
-
6
- Distribute tests over many workers using a queue.
7
-
8
- ## Why a queue?
9
-
10
- One big problem with distributed test suites, is test imbalance. Meaning that one worker would spend 10 minutes while all the others are done after 1 minute.
11
- There is algorithms available to balance perfectly your workers, but in practice your test performance tend to vary, and it's easier to consider tests as work unit in a queue and let workers pop them as fast as possible.
12
-
13
- Another advantage is that if you lose workers along the way, using a queue the other workers can pick up the job, making you resilient to failures.
14
-
15
- ## Installation
16
-
17
- Add this line to your application's Gemfile:
18
-
19
- ```ruby
20
- gem 'ci-queue'
21
- ```
22
-
23
- And then execute:
24
-
25
- $ bundle
26
-
27
- Or install it yourself as:
28
-
29
- $ gem install ci-queue
30
-
31
- ## Usage
32
-
33
- TODO: Write usage instructions here
34
-
35
- ## Development
36
-
37
- After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
38
-
39
- To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
40
-
41
- ## Contributing
42
-
43
- Bug reports and pull requests are welcome on GitHub at https://github.com/Shopify/ci-queue.
44
-
45
- ## License
46
-
47
- The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
48
-