ci-queue 0.9.1 → 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 75fc560a4dbae8a7ad7487570afaef29048788b8
4
- data.tar.gz: 1f84cff31f94272e01e436e70a3f4a66a9dc70d5
3
+ metadata.gz: c56cc78cb306fb67c398a2cdf41b5a9d7d594d87
4
+ data.tar.gz: d63fb0d78ec9fe9eeb04bc2bc60a1ab6aeeab77d
5
5
  SHA512:
6
- metadata.gz: af05def95dc288e9a78b9fa5ccae1d62080d74a37f50ed3702cbceefc7da6ea641178dd988d1b02e1ae574bb57aa07e4fcfa7a6d4f434e79f86c23248623efe2
7
- data.tar.gz: 18930ed5ca16d136100658302e23509062245259cc3e7cc2749c9c4e7812e7a28c692799cfaad85e982aec8bd34b5593c4f898d8e5851550a53f2cc738b1f32c
6
+ metadata.gz: 4ef344549d3bb3e75bb01f0e9bae693b2ed635eedcc31f5043383de2c8775e662c1822ba740435b174e0f9ebce3346b97ccc203a44ce433ab922ab9796d2a91f
7
+ data.tar.gz: 0b6ec24e92bf4ec95d6bd68a20e64efab06dcf5109bb9c7b3ab1ad2f8632cc8907865dbaacebff891606f817d0101376281ba5549d152b8acfc3ebfe537eeea9
@@ -35,4 +35,7 @@ Gem::Specification.new do |spec|
35
35
  spec.add_development_dependency 'redis', '~> 3.3'
36
36
  spec.add_development_dependency 'simplecov', '~> 0.12'
37
37
  spec.add_development_dependency 'minitest-reporters', '~> 1.1'
38
+
39
+ spec.add_development_dependency 'snappy'
40
+ spec.add_development_dependency 'msgpack'
38
41
  end
@@ -3,7 +3,6 @@ require 'cgi'
3
3
 
4
4
  require 'ci/queue/version'
5
5
  require 'ci/queue/output_helpers'
6
- require 'ci/queue/index'
7
6
  require 'ci/queue/configuration'
8
7
  require 'ci/queue/static'
9
8
  require 'ci/queue/file'
@@ -13,6 +12,16 @@ module CI
13
12
  module Queue
14
13
  extend self
15
14
 
15
+ attr_accessor :shuffler
16
+
17
+ def shuffle(tests, random)
18
+ if shuffler
19
+ shuffler.call(tests, random)
20
+ else
21
+ tests.sort.shuffle(random: random)
22
+ end
23
+ end
24
+
16
25
  def from_uri(url, config)
17
26
  uri = URI(url)
18
27
  implementation = case uri.scheme
@@ -10,9 +10,8 @@ module CI
10
10
  @tests.size
11
11
  end
12
12
 
13
- def populate(all_tests, &test_indexer)
13
+ def populate(all_tests, random: nil)
14
14
  @all_tests = all_tests
15
- @test_indexer = test_indexer
16
15
  end
17
16
 
18
17
  def to_a
@@ -24,11 +23,11 @@ module CI
24
23
  end
25
24
 
26
25
  def failing_test
27
- Static.new([config.failing_test], config).populate(@all_tests, &@test_indexer)
26
+ Static.new([config.failing_test], config).populate(@all_tests)
28
27
  end
29
28
 
30
29
  def candidates
31
- Static.new(first_half + [config.failing_test], config).populate(@all_tests, &@test_indexer)
30
+ Static.new(first_half + [config.failing_test], config).populate(@all_tests)
32
31
  end
33
32
 
34
33
  def failed!
@@ -19,9 +19,10 @@ module CI
19
19
  super(redis, config)
20
20
  end
21
21
 
22
- def populate(tests, &indexer)
23
- @index = Index.new(tests, &indexer)
24
- push(tests.map { |t| index.key(t) })
22
+ def populate(tests, random: Random.new)
23
+ @index = tests.map { |t| [t.id, t] }.to_h
24
+ tests = Queue.shuffle(tests, random)
25
+ push(tests.map(&:id))
25
26
  self
26
27
  end
27
28
 
@@ -76,7 +77,7 @@ module CI
76
77
  end
77
78
 
78
79
  def acknowledge(test)
79
- test_key = index.key(test)
80
+ test_key = test.id
80
81
  raise_on_mismatching_test(test_key)
81
82
  eval_script(
82
83
  :acknowledge,
@@ -86,7 +87,7 @@ module CI
86
87
  end
87
88
 
88
89
  def requeue(test, offset: Redis.requeue_offset)
89
- test_key = index.key(test)
90
+ test_key = test.id
90
91
  raise_on_mismatching_test(test_key)
91
92
 
92
93
  requeued = eval_script(
@@ -32,8 +32,8 @@ module CI
32
32
  self
33
33
  end
34
34
 
35
- def populate(tests, &indexer)
36
- @index = Index.new(tests, &indexer)
35
+ def populate(tests, random: nil)
36
+ @index = tests.map { |t| [t.id, t] }.to_h
37
37
  self
38
38
  end
39
39
 
@@ -65,10 +65,10 @@ module CI
65
65
  end
66
66
 
67
67
  def requeue(test)
68
- key = index.key(test)
69
- return false unless should_requeue?(key)
70
- requeues[key] += 1
71
- @queue.unshift(index.key(test))
68
+ test_key = test.id
69
+ return false unless should_requeue?(test_key)
70
+ requeues[test_key] += 1
71
+ @queue.unshift(test_key)
72
72
  true
73
73
  end
74
74
 
@@ -1,6 +1,6 @@
1
1
  module CI
2
2
  module Queue
3
- VERSION = '0.9.1'
3
+ VERSION = '0.9.2'
4
4
  DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
5
5
  RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
6
6
  end
@@ -45,6 +45,25 @@ module Minitest
45
45
  end
46
46
 
47
47
  module Queue
48
+ class SingleExample
49
+ def initialize(runnable, method_name)
50
+ @runnable = runnable
51
+ @method_name = method_name
52
+ end
53
+
54
+ def id
55
+ @id ||= "#{@runnable}##{@method_name}"
56
+ end
57
+
58
+ def <=>(other)
59
+ id <=> other.id
60
+ end
61
+
62
+ def run
63
+ Minitest.run_one_method(@runnable, @method_name)
64
+ end
65
+ end
66
+
48
67
  attr_reader :queue
49
68
 
50
69
  def queue=(queue)
@@ -63,12 +82,10 @@ module Minitest
63
82
  @queue_reporters = reporters
64
83
  end
65
84
 
66
- SuiteNotFound = Class.new(StandardError)
67
-
68
85
  def loaded_tests
69
- Minitest::Test.runnables.flat_map do |suite|
70
- suite.runnable_methods.map do |method|
71
- "#{suite}##{method}"
86
+ Minitest::Test.runnables.flat_map do |runnable|
87
+ runnable.runnable_methods.map do |method_name|
88
+ SingleExample.new(runnable, method_name)
72
89
  end
73
90
  end
74
91
  end
@@ -82,24 +99,16 @@ module Minitest
82
99
  end
83
100
 
84
101
  def run_from_queue(reporter, *)
85
- runnable_classes = Minitest::Runnable.runnables.map { |s| [s.name, s] }.to_h
86
-
87
- queue.poll do |test_name|
88
- class_name, method_name = test_name.split("#".freeze, 2)
89
-
90
- if klass = runnable_classes[class_name]
91
- result = Minitest.run_one_method(klass, method_name)
92
- failed = !(result.passed? || result.skipped?)
93
- if failed && queue.requeue(test_name)
94
- result.requeue!
95
- reporter.record(result)
96
- elsif queue.acknowledge(test_name) || !failed
97
- # If the test was already acknowledged by another worker (we timed out)
98
- # Then we only record it if it is successful.
99
- reporter.record(result)
100
- end
101
- else
102
- raise SuiteNotFound, "Couldn't find suite matching: #{test_name}"
102
+ queue.poll do |example|
103
+ result = example.run
104
+ failed = !(result.passed? || result.skipped?)
105
+ if failed && queue.requeue(example)
106
+ result.requeue!
107
+ reporter.record(result)
108
+ elsif queue.acknowledge(example) || !failed
109
+ # If the test was already acknowledged by another worker (we timed out)
110
+ # Then we only record it if it is successful.
111
+ reporter.record(result)
103
112
  end
104
113
  end
105
114
  end
@@ -89,7 +89,7 @@ module Minitest
89
89
  command += argv
90
90
 
91
91
  puts
92
- puts "cat <<EOF |\n#{failing_order.to_a.join("\n")}\nEOF\n#{command.join(' ')}"
92
+ puts "cat <<EOF |\n#{failing_order.to_a.map(&:id).join("\n")}\nEOF\n#{command.join(' ')}"
93
93
  puts
94
94
  exit! 0
95
95
  end
@@ -139,7 +139,7 @@ module Minitest
139
139
  end
140
140
 
141
141
  def populate_queue
142
- Minitest.queue.populate(shuffle(Minitest.loaded_tests), &:to_s) # TODO: stop serializing
142
+ Minitest.queue.populate(Minitest.loaded_tests, random: ordering_seed, &:id)
143
143
  end
144
144
 
145
145
  def set_load_path
@@ -283,10 +283,12 @@ module Minitest
283
283
  string.lines.map(&:strip)
284
284
  end
285
285
 
286
- def shuffle(tests)
287
- return tests unless queue_config.seed
288
- random = Random.new(Digest::MD5.hexdigest(queue_config.seed).to_i(16))
289
- tests.shuffle(random: random)
286
+ def ordering_seed
287
+ if queue_config.seed
288
+ Random.new(Digest::MD5.hexdigest(queue_config.seed).to_i(16))
289
+ else
290
+ Random.new
291
+ end
290
292
  end
291
293
 
292
294
  def queue_url
@@ -31,6 +31,36 @@ module Minitest
31
31
 
32
32
  self.coder = Marshal
33
33
 
34
+ begin
35
+ require 'snappy'
36
+ require 'msgpack'
37
+ require 'stringio'
38
+
39
+ module SnappyPack
40
+ extend self
41
+
42
+ MSGPACK = MessagePack::Factory.new
43
+ MSGPACK.register_type(0x00, Symbol)
44
+
45
+ def load(payload)
46
+ io = StringIO.new(Snappy.inflate(payload))
47
+ MSGPACK.unpacker(io).unpack
48
+ end
49
+
50
+ def dump(object)
51
+ io = StringIO.new
52
+ packer = MSGPACK.packer(io)
53
+ packer.pack(object)
54
+ packer.flush
55
+ io.rewind
56
+ Snappy.deflate(io.string).force_encoding(Encoding::UTF_8)
57
+ end
58
+ end
59
+
60
+ self.coder = SnappyPack
61
+ rescue LoadError
62
+ end
63
+
34
64
  def initialize(data)
35
65
  @data = data
36
66
  end
@@ -126,6 +126,10 @@ module RSpec
126
126
  example.id
127
127
  end
128
128
 
129
+ def <=>(other)
130
+ id <=> other.id
131
+ end
132
+
129
133
  def run(reporter)
130
134
  return if RSpec.world.wants_to_quit
131
135
  instance = example_group.new(example.inspect_output)
@@ -156,7 +160,7 @@ module RSpec
156
160
  end
157
161
 
158
162
  queue = CI::Queue.from_uri(queue_url, RSpec::Queue.config)
159
- queue.populate(shuffle(examples), &:id)
163
+ queue.populate(examples, random: ordering_seed, &:id)
160
164
  examples_count = examples.size # TODO: figure out which stub value would be best
161
165
  success = true
162
166
  @configuration.reporter.report(examples_count) do |reporter|
@@ -174,10 +178,12 @@ module RSpec
174
178
 
175
179
  private
176
180
 
177
- def shuffle(examples)
178
- return examples unless RSpec::Queue.config.seed
179
- random = Random.new(Digest::MD5.hexdigest(RSpec::Queue.config.seed).to_i(16))
180
- examples.shuffle(random: random)
181
+ def ordering_seed
182
+ if RSpec::Queue.config.seed
183
+ Random.new(Digest::MD5.hexdigest(RSpec::Queue.config.seed).to_i(16))
184
+ else
185
+ Random.new
186
+ end
181
187
  end
182
188
 
183
189
  def queue_url
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ci-queue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.1
4
+ version: 0.9.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jean Boussier
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-11-30 00:00:00.000000000 Z
11
+ date: 2017-12-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: ansi
@@ -122,6 +122,34 @@ dependencies:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
124
  version: '1.1'
125
+ - !ruby/object:Gem::Dependency
126
+ name: snappy
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: msgpack
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
125
153
  description: To parallelize your CI without having to balance your tests
126
154
  email:
127
155
  - jean.boussier@shopify.com
@@ -147,7 +175,6 @@ files:
147
175
  - lib/ci/queue/bisect.rb
148
176
  - lib/ci/queue/configuration.rb
149
177
  - lib/ci/queue/file.rb
150
- - lib/ci/queue/index.rb
151
178
  - lib/ci/queue/output_helpers.rb
152
179
  - lib/ci/queue/redis.rb
153
180
  - lib/ci/queue/redis/acknowledge.lua
@@ -1,20 +0,0 @@
1
- module CI
2
- module Queue
3
- class Index
4
- def initialize(objects, &indexer)
5
- @index = objects.map { |o| [indexer.call(o), o] }.to_h
6
- @indexer = indexer
7
- end
8
-
9
- def fetch(key)
10
- @index.fetch(key)
11
- end
12
-
13
- def key(value)
14
- key = @indexer.call(value)
15
- raise KeyError, "value not found: #{value.inspect}" unless @index.key?(key)
16
- key
17
- end
18
- end
19
- end
20
- end