ci-queue 0.9.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ci-queue.gemspec +3 -0
- data/lib/ci/queue.rb +10 -1
- data/lib/ci/queue/bisect.rb +3 -4
- data/lib/ci/queue/redis/worker.rb +6 -5
- data/lib/ci/queue/static.rb +6 -6
- data/lib/ci/queue/version.rb +1 -1
- data/lib/minitest/queue.rb +32 -23
- data/lib/minitest/queue/runner.rb +8 -6
- data/lib/minitest/reporters/redis_reporter.rb +30 -0
- data/lib/rspec/queue.rb +11 -5
- metadata +30 -3
- data/lib/ci/queue/index.rb +0 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c56cc78cb306fb67c398a2cdf41b5a9d7d594d87
|
4
|
+
data.tar.gz: d63fb0d78ec9fe9eeb04bc2bc60a1ab6aeeab77d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4ef344549d3bb3e75bb01f0e9bae693b2ed635eedcc31f5043383de2c8775e662c1822ba740435b174e0f9ebce3346b97ccc203a44ce433ab922ab9796d2a91f
|
7
|
+
data.tar.gz: 0b6ec24e92bf4ec95d6bd68a20e64efab06dcf5109bb9c7b3ab1ad2f8632cc8907865dbaacebff891606f817d0101376281ba5549d152b8acfc3ebfe537eeea9
|
data/ci-queue.gemspec
CHANGED
@@ -35,4 +35,7 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.add_development_dependency 'redis', '~> 3.3'
|
36
36
|
spec.add_development_dependency 'simplecov', '~> 0.12'
|
37
37
|
spec.add_development_dependency 'minitest-reporters', '~> 1.1'
|
38
|
+
|
39
|
+
spec.add_development_dependency 'snappy'
|
40
|
+
spec.add_development_dependency 'msgpack'
|
38
41
|
end
|
data/lib/ci/queue.rb
CHANGED
@@ -3,7 +3,6 @@ require 'cgi'
|
|
3
3
|
|
4
4
|
require 'ci/queue/version'
|
5
5
|
require 'ci/queue/output_helpers'
|
6
|
-
require 'ci/queue/index'
|
7
6
|
require 'ci/queue/configuration'
|
8
7
|
require 'ci/queue/static'
|
9
8
|
require 'ci/queue/file'
|
@@ -13,6 +12,16 @@ module CI
|
|
13
12
|
module Queue
|
14
13
|
extend self
|
15
14
|
|
15
|
+
attr_accessor :shuffler
|
16
|
+
|
17
|
+
def shuffle(tests, random)
|
18
|
+
if shuffler
|
19
|
+
shuffler.call(tests, random)
|
20
|
+
else
|
21
|
+
tests.sort.shuffle(random: random)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
16
25
|
def from_uri(url, config)
|
17
26
|
uri = URI(url)
|
18
27
|
implementation = case uri.scheme
|
data/lib/ci/queue/bisect.rb
CHANGED
@@ -10,9 +10,8 @@ module CI
|
|
10
10
|
@tests.size
|
11
11
|
end
|
12
12
|
|
13
|
-
def populate(all_tests,
|
13
|
+
def populate(all_tests, random: nil)
|
14
14
|
@all_tests = all_tests
|
15
|
-
@test_indexer = test_indexer
|
16
15
|
end
|
17
16
|
|
18
17
|
def to_a
|
@@ -24,11 +23,11 @@ module CI
|
|
24
23
|
end
|
25
24
|
|
26
25
|
def failing_test
|
27
|
-
Static.new([config.failing_test], config).populate(@all_tests
|
26
|
+
Static.new([config.failing_test], config).populate(@all_tests)
|
28
27
|
end
|
29
28
|
|
30
29
|
def candidates
|
31
|
-
Static.new(first_half + [config.failing_test], config).populate(@all_tests
|
30
|
+
Static.new(first_half + [config.failing_test], config).populate(@all_tests)
|
32
31
|
end
|
33
32
|
|
34
33
|
def failed!
|
@@ -19,9 +19,10 @@ module CI
|
|
19
19
|
super(redis, config)
|
20
20
|
end
|
21
21
|
|
22
|
-
def populate(tests,
|
23
|
-
@index =
|
24
|
-
|
22
|
+
def populate(tests, random: Random.new)
|
23
|
+
@index = tests.map { |t| [t.id, t] }.to_h
|
24
|
+
tests = Queue.shuffle(tests, random)
|
25
|
+
push(tests.map(&:id))
|
25
26
|
self
|
26
27
|
end
|
27
28
|
|
@@ -76,7 +77,7 @@ module CI
|
|
76
77
|
end
|
77
78
|
|
78
79
|
def acknowledge(test)
|
79
|
-
test_key =
|
80
|
+
test_key = test.id
|
80
81
|
raise_on_mismatching_test(test_key)
|
81
82
|
eval_script(
|
82
83
|
:acknowledge,
|
@@ -86,7 +87,7 @@ module CI
|
|
86
87
|
end
|
87
88
|
|
88
89
|
def requeue(test, offset: Redis.requeue_offset)
|
89
|
-
test_key =
|
90
|
+
test_key = test.id
|
90
91
|
raise_on_mismatching_test(test_key)
|
91
92
|
|
92
93
|
requeued = eval_script(
|
data/lib/ci/queue/static.rb
CHANGED
@@ -32,8 +32,8 @@ module CI
|
|
32
32
|
self
|
33
33
|
end
|
34
34
|
|
35
|
-
def populate(tests,
|
36
|
-
@index =
|
35
|
+
def populate(tests, random: nil)
|
36
|
+
@index = tests.map { |t| [t.id, t] }.to_h
|
37
37
|
self
|
38
38
|
end
|
39
39
|
|
@@ -65,10 +65,10 @@ module CI
|
|
65
65
|
end
|
66
66
|
|
67
67
|
def requeue(test)
|
68
|
-
|
69
|
-
return false unless should_requeue?(
|
70
|
-
requeues[
|
71
|
-
@queue.unshift(
|
68
|
+
test_key = test.id
|
69
|
+
return false unless should_requeue?(test_key)
|
70
|
+
requeues[test_key] += 1
|
71
|
+
@queue.unshift(test_key)
|
72
72
|
true
|
73
73
|
end
|
74
74
|
|
data/lib/ci/queue/version.rb
CHANGED
data/lib/minitest/queue.rb
CHANGED
@@ -45,6 +45,25 @@ module Minitest
|
|
45
45
|
end
|
46
46
|
|
47
47
|
module Queue
|
48
|
+
class SingleExample
|
49
|
+
def initialize(runnable, method_name)
|
50
|
+
@runnable = runnable
|
51
|
+
@method_name = method_name
|
52
|
+
end
|
53
|
+
|
54
|
+
def id
|
55
|
+
@id ||= "#{@runnable}##{@method_name}"
|
56
|
+
end
|
57
|
+
|
58
|
+
def <=>(other)
|
59
|
+
id <=> other.id
|
60
|
+
end
|
61
|
+
|
62
|
+
def run
|
63
|
+
Minitest.run_one_method(@runnable, @method_name)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
48
67
|
attr_reader :queue
|
49
68
|
|
50
69
|
def queue=(queue)
|
@@ -63,12 +82,10 @@ module Minitest
|
|
63
82
|
@queue_reporters = reporters
|
64
83
|
end
|
65
84
|
|
66
|
-
SuiteNotFound = Class.new(StandardError)
|
67
|
-
|
68
85
|
def loaded_tests
|
69
|
-
Minitest::Test.runnables.flat_map do |
|
70
|
-
|
71
|
-
|
86
|
+
Minitest::Test.runnables.flat_map do |runnable|
|
87
|
+
runnable.runnable_methods.map do |method_name|
|
88
|
+
SingleExample.new(runnable, method_name)
|
72
89
|
end
|
73
90
|
end
|
74
91
|
end
|
@@ -82,24 +99,16 @@ module Minitest
|
|
82
99
|
end
|
83
100
|
|
84
101
|
def run_from_queue(reporter, *)
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
if
|
94
|
-
|
95
|
-
reporter.record(result)
|
96
|
-
elsif queue.acknowledge(test_name) || !failed
|
97
|
-
# If the test was already acknowledged by another worker (we timed out)
|
98
|
-
# Then we only record it if it is successful.
|
99
|
-
reporter.record(result)
|
100
|
-
end
|
101
|
-
else
|
102
|
-
raise SuiteNotFound, "Couldn't find suite matching: #{test_name}"
|
102
|
+
queue.poll do |example|
|
103
|
+
result = example.run
|
104
|
+
failed = !(result.passed? || result.skipped?)
|
105
|
+
if failed && queue.requeue(example)
|
106
|
+
result.requeue!
|
107
|
+
reporter.record(result)
|
108
|
+
elsif queue.acknowledge(example) || !failed
|
109
|
+
# If the test was already acknowledged by another worker (we timed out)
|
110
|
+
# Then we only record it if it is successful.
|
111
|
+
reporter.record(result)
|
103
112
|
end
|
104
113
|
end
|
105
114
|
end
|
@@ -89,7 +89,7 @@ module Minitest
|
|
89
89
|
command += argv
|
90
90
|
|
91
91
|
puts
|
92
|
-
puts "cat <<EOF |\n#{failing_order.to_a.join("\n")}\nEOF\n#{command.join(' ')}"
|
92
|
+
puts "cat <<EOF |\n#{failing_order.to_a.map(&:id).join("\n")}\nEOF\n#{command.join(' ')}"
|
93
93
|
puts
|
94
94
|
exit! 0
|
95
95
|
end
|
@@ -139,7 +139,7 @@ module Minitest
|
|
139
139
|
end
|
140
140
|
|
141
141
|
def populate_queue
|
142
|
-
Minitest.queue.populate(
|
142
|
+
Minitest.queue.populate(Minitest.loaded_tests, random: ordering_seed, &:id)
|
143
143
|
end
|
144
144
|
|
145
145
|
def set_load_path
|
@@ -283,10 +283,12 @@ module Minitest
|
|
283
283
|
string.lines.map(&:strip)
|
284
284
|
end
|
285
285
|
|
286
|
-
def
|
287
|
-
|
288
|
-
|
289
|
-
|
286
|
+
def ordering_seed
|
287
|
+
if queue_config.seed
|
288
|
+
Random.new(Digest::MD5.hexdigest(queue_config.seed).to_i(16))
|
289
|
+
else
|
290
|
+
Random.new
|
291
|
+
end
|
290
292
|
end
|
291
293
|
|
292
294
|
def queue_url
|
@@ -31,6 +31,36 @@ module Minitest
|
|
31
31
|
|
32
32
|
self.coder = Marshal
|
33
33
|
|
34
|
+
begin
|
35
|
+
require 'snappy'
|
36
|
+
require 'msgpack'
|
37
|
+
require 'stringio'
|
38
|
+
|
39
|
+
module SnappyPack
|
40
|
+
extend self
|
41
|
+
|
42
|
+
MSGPACK = MessagePack::Factory.new
|
43
|
+
MSGPACK.register_type(0x00, Symbol)
|
44
|
+
|
45
|
+
def load(payload)
|
46
|
+
io = StringIO.new(Snappy.inflate(payload))
|
47
|
+
MSGPACK.unpacker(io).unpack
|
48
|
+
end
|
49
|
+
|
50
|
+
def dump(object)
|
51
|
+
io = StringIO.new
|
52
|
+
packer = MSGPACK.packer(io)
|
53
|
+
packer.pack(object)
|
54
|
+
packer.flush
|
55
|
+
io.rewind
|
56
|
+
Snappy.deflate(io.string).force_encoding(Encoding::UTF_8)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
self.coder = SnappyPack
|
61
|
+
rescue LoadError
|
62
|
+
end
|
63
|
+
|
34
64
|
def initialize(data)
|
35
65
|
@data = data
|
36
66
|
end
|
data/lib/rspec/queue.rb
CHANGED
@@ -126,6 +126,10 @@ module RSpec
|
|
126
126
|
example.id
|
127
127
|
end
|
128
128
|
|
129
|
+
def <=>(other)
|
130
|
+
id <=> other.id
|
131
|
+
end
|
132
|
+
|
129
133
|
def run(reporter)
|
130
134
|
return if RSpec.world.wants_to_quit
|
131
135
|
instance = example_group.new(example.inspect_output)
|
@@ -156,7 +160,7 @@ module RSpec
|
|
156
160
|
end
|
157
161
|
|
158
162
|
queue = CI::Queue.from_uri(queue_url, RSpec::Queue.config)
|
159
|
-
queue.populate(
|
163
|
+
queue.populate(examples, random: ordering_seed, &:id)
|
160
164
|
examples_count = examples.size # TODO: figure out which stub value would be best
|
161
165
|
success = true
|
162
166
|
@configuration.reporter.report(examples_count) do |reporter|
|
@@ -174,10 +178,12 @@ module RSpec
|
|
174
178
|
|
175
179
|
private
|
176
180
|
|
177
|
-
def
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
+
def ordering_seed
|
182
|
+
if RSpec::Queue.config.seed
|
183
|
+
Random.new(Digest::MD5.hexdigest(RSpec::Queue.config.seed).to_i(16))
|
184
|
+
else
|
185
|
+
Random.new
|
186
|
+
end
|
181
187
|
end
|
182
188
|
|
183
189
|
def queue_url
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ci-queue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jean Boussier
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-12-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: ansi
|
@@ -122,6 +122,34 @@ dependencies:
|
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '1.1'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: snappy
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: msgpack
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - ">="
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '0'
|
146
|
+
type: :development
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '0'
|
125
153
|
description: To parallelize your CI without having to balance your tests
|
126
154
|
email:
|
127
155
|
- jean.boussier@shopify.com
|
@@ -147,7 +175,6 @@ files:
|
|
147
175
|
- lib/ci/queue/bisect.rb
|
148
176
|
- lib/ci/queue/configuration.rb
|
149
177
|
- lib/ci/queue/file.rb
|
150
|
-
- lib/ci/queue/index.rb
|
151
178
|
- lib/ci/queue/output_helpers.rb
|
152
179
|
- lib/ci/queue/redis.rb
|
153
180
|
- lib/ci/queue/redis/acknowledge.lua
|
data/lib/ci/queue/index.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
module CI
|
2
|
-
module Queue
|
3
|
-
class Index
|
4
|
-
def initialize(objects, &indexer)
|
5
|
-
@index = objects.map { |o| [indexer.call(o), o] }.to_h
|
6
|
-
@indexer = indexer
|
7
|
-
end
|
8
|
-
|
9
|
-
def fetch(key)
|
10
|
-
@index.fetch(key)
|
11
|
-
end
|
12
|
-
|
13
|
-
def key(value)
|
14
|
-
key = @indexer.call(value)
|
15
|
-
raise KeyError, "value not found: #{value.inspect}" unless @index.key?(key)
|
16
|
-
key
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|