ci-queue 0.22.0 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a727d6a5f5c01a4b5837d1f7ed629ce3f9c971fed502d6b850a43b384e72c59a
4
- data.tar.gz: 3ab46419fd8c1932d2e94cd40bbfb087df927d9013f9d0e998f3ed60839440e6
3
+ metadata.gz: aaa8a7cb585452d354e884d29772f29e0ea6dda29e894a4e69088ab302cec1b7
4
+ data.tar.gz: 8dc28379b0488fce6b951a0bf5ac434cce71fa5751c453f807ee6412edc86ef4
5
5
  SHA512:
6
- metadata.gz: f6fd6c5f71fc10b0b5085b0d5259aa25eb1b1cc969f7b182e45c975deff56e72b1bef6e133adb812db87809feda33044ad7e137f36d88c5866c34a469af4b0fd
7
- data.tar.gz: 0415f91c5aa6c1bdfd185842ae88ee5d73ed3c553945eeacd2a095288156336f4bb00d2d49ab302f7758d7fcbb78c7fcd2b500d6265c12aa3b33dfbc2d5d7829
6
+ metadata.gz: b2fe2f4bfd08b4165576b2f881fdad537b22566fe8c940ee87a03e1ecb13488398b3a7e86147f83f284603c007cffbd606f8576b5f5a17a7d7c034f7dbacdf41
7
+ data.tar.gz: d12edaffb47e719b0c9e437da5261c4808dac54f2fa1e080d857c36225550a3f3a6d20517fe54e0da3ddaf1cb6d965fa6e666866049564ce741f6573aa187ff5
data/README.md CHANGED
@@ -70,3 +70,9 @@ rspec-queue --queue redis://example.com --timeout 600 --report
70
70
  #### Limitations
71
71
 
72
72
  Because of how `ci-queue` executes the examples, `before(:all)` and `after(:all)` hooks are not supported. `rspec-queue` will explicitly reject them.
73
+
74
+ ## Custom Redis Expiry
75
+
76
+ `ci-queue` expects the Redis server to have an [eviction policy](https://redis.io/docs/manual/eviction/#eviction-policies) of `allkeys-lru`.
77
+
78
+ You can also use `--redis-ttl` to set a custom expiration time for all CI Queue keys, this defaults to 8 hours (28,800 seconds)
data/ci-queue.gemspec CHANGED
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
32
32
  spec.add_development_dependency 'bundler'
33
33
  spec.add_development_dependency 'rake'
34
34
  spec.add_development_dependency 'minitest', ENV.fetch('MINITEST_VERSION', '~> 5.11')
35
- spec.add_development_dependency 'rspec', '~> 3.7.0'
35
+ spec.add_development_dependency 'rspec', '~> 3.10'
36
36
  spec.add_development_dependency 'redis'
37
37
  spec.add_development_dependency 'simplecov', '~> 0.12'
38
38
  spec.add_development_dependency 'minitest-reporters', '~> 1.1'
@@ -5,7 +5,7 @@ module CI
5
5
  attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file
6
6
  attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
7
7
  attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
8
- attr_accessor :max_test_failed
8
+ attr_accessor :max_test_failed, :redis_ttl
9
9
  attr_reader :circuit_breakers
10
10
  attr_writer :seed, :build_id
11
11
  attr_writer :queue_init_timeout
@@ -18,6 +18,7 @@ module CI
18
18
  seed: env['CIRCLE_SHA1'] || env['BUILDKITE_COMMIT'] || env['TRAVIS_COMMIT'] || env['HEROKU_TEST_RUN_COMMIT_VERSION'] || env['SEMAPHORE_GIT_SHA'],
19
19
  flaky_tests: load_flaky_tests(env['CI_QUEUE_FLAKY_TESTS']),
20
20
  statsd_endpoint: env['CI_QUEUE_STATSD_ADDR'],
21
+ redis_ttl: env['CI_QUEUE_REDIS_TTL']&.to_i || 8 * 60 * 60,
21
22
  )
22
23
  end
23
24
 
@@ -34,7 +35,7 @@ module CI
34
35
  namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
35
36
  grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
36
37
  max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
37
- queue_init_timeout: nil
38
+ queue_init_timeout: nil, redis_ttl: 8 * 60 * 60
38
39
  )
39
40
  @build_id = build_id
40
41
  @circuit_breakers = [CircuitBreaker::Disabled]
@@ -55,6 +56,7 @@ module CI
55
56
  @worker_id = worker_id
56
57
  self.max_consecutive_failures = max_consecutive_failures
57
58
  self.max_duration = max_duration
59
+ @redis_ttl = redis_ttl
58
60
  end
59
61
 
60
62
  def queue_init_timeout
@@ -21,16 +21,16 @@ module CI
21
21
  end
22
22
 
23
23
  def size
24
- redis.multi do
25
- redis.llen(key('queue'))
26
- redis.zcard(key('running'))
24
+ redis.multi do |transaction|
25
+ transaction.llen(key('queue'))
26
+ transaction.zcard(key('running'))
27
27
  end.inject(:+)
28
28
  end
29
29
 
30
30
  def to_a
31
- redis.multi do
32
- redis.lrange(key('queue'), 0, -1)
33
- redis.zrange(key('running'), 0, -1)
31
+ redis.multi do |transaction|
32
+ transaction.lrange(key('queue'), 0, -1)
33
+ transaction.zrange(key('running'), 0, -1)
34
34
  end.flatten.reverse.map { |k| index.fetch(k) }
35
35
  end
36
36
 
@@ -22,9 +22,9 @@ module CI
22
22
  end
23
23
 
24
24
  def pop_warnings
25
- warnings = redis.multi do
26
- redis.lrange(key('warnings'), 0, -1)
27
- redis.del(key('warnings'))
25
+ warnings = redis.multi do |transaction|
26
+ transaction.lrange(key('warnings'), 0, -1)
27
+ transaction.del(key('warnings'))
28
28
  end.first
29
29
 
30
30
  warnings.map { |p| Marshal.load(p) }
@@ -35,21 +35,22 @@ module CI
35
35
  end
36
36
 
37
37
  def record_error(id, payload, stats: nil)
38
- redis.pipelined do
39
- redis.hset(
38
+ redis.pipelined do |pipeline|
39
+ pipeline.hset(
40
40
  key('error-reports'),
41
41
  id.dup.force_encoding(Encoding::BINARY),
42
42
  payload.dup.force_encoding(Encoding::BINARY),
43
43
  )
44
- record_stats(stats)
44
+ pipeline.expire(key('error-reports'), config.redis_ttl)
45
+ record_stats(stats, pipeline: pipeline)
45
46
  end
46
47
  nil
47
48
  end
48
49
 
49
50
  def record_success(id, stats: nil)
50
- redis.pipelined do
51
- redis.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
52
- record_stats(stats)
51
+ redis.pipelined do |pipeline|
52
+ pipeline.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
53
+ record_stats(stats, pipeline: pipeline)
53
54
  end
54
55
  nil
55
56
  end
@@ -65,8 +66,8 @@ module CI
65
66
  end
66
67
 
67
68
  def fetch_stats(stat_names)
68
- counts = redis.pipelined do
69
- stat_names.each { |c| redis.hvals(key(c)) }
69
+ counts = redis.pipelined do |pipeline|
70
+ stat_names.each { |c| pipeline.hvals(key(c)) }
70
71
  end
71
72
  sum_counts = counts.map do |values|
72
73
  values.map(&:to_f).inject(:+).to_f
@@ -75,9 +76,9 @@ module CI
75
76
  end
76
77
 
77
78
  def reset_stats(stat_names)
78
- redis.pipelined do
79
+ redis.pipelined do |pipeline|
79
80
  stat_names.each do |stat_name|
80
- redis.hdel(key(stat_name), config.worker_id)
81
+ pipeline.hdel(key(stat_name), config.worker_id)
81
82
  end
82
83
  end
83
84
  end
@@ -86,10 +87,11 @@ module CI
86
87
 
87
88
  attr_reader :config, :redis
88
89
 
89
- def record_stats(stats)
90
+ def record_stats(stats, pipeline: redis)
90
91
  return unless stats
91
92
  stats.each do |stat_name, stat_value|
92
- redis.hset(key(stat_name), config.worker_id, stat_value)
93
+ pipeline.hset(key(stat_name), config.worker_id, stat_value)
94
+ pipeline.expire(key(stat_name), config.redis_ttl)
93
95
  end
94
96
  end
95
97
 
@@ -11,12 +11,13 @@ module CI
11
11
  end
12
12
 
13
13
  def record_error(payload, stats: nil)
14
- redis.pipelined do
15
- redis.lpush(
14
+ redis.pipelined do |pipeline|
15
+ pipeline.lpush(
16
16
  key('error-reports'),
17
17
  payload.force_encoding(Encoding::BINARY),
18
18
  )
19
- record_stats(stats)
19
+ pipeline.expire(key('error-reports'), config.redis_ttl)
20
+ record_stats(stats, pipeline: pipeline)
20
21
  end
21
22
  nil
22
23
  end
@@ -34,8 +35,8 @@ module CI
34
35
  end
35
36
 
36
37
  def fetch_stats(stat_names)
37
- counts = redis.pipelined do
38
- stat_names.each { |c| redis.hvals(key(c)) }
38
+ counts = redis.pipelined do |pipeline|
39
+ stat_names.each { |c| pipeline.hvals(key(c)) }
39
40
  end
40
41
  stat_names.zip(counts.map { |values| values.map(&:to_f).inject(:+).to_f }).to_h
41
42
  end
@@ -54,10 +55,11 @@ module CI
54
55
  ['build', config.build_id, *args].join(':')
55
56
  end
56
57
 
57
- def record_stats(stats)
58
+ def record_stats(stats, pipeline: redis)
58
59
  return unless stats
59
60
  stats.each do |stat_name, stat_value|
60
- redis.hset(key(stat_name), config.worker_id, stat_value)
61
+ pipeline.hset(key(stat_name), config.worker_id, stat_value)
62
+ pipeline.expire(key(stat_name), config.redis_ttl)
61
63
  end
62
64
  end
63
65
  end
@@ -19,38 +19,37 @@ module CI
19
19
  attr_reader :redis
20
20
 
21
21
  def record_test_time(test_name, duration)
22
- redis.pipelined do
23
- redis.lpush(
22
+ redis.pipelined do |pipeline|
23
+ pipeline.lpush(
24
24
  test_time_key(test_name),
25
25
  duration.to_s.force_encoding(Encoding::BINARY),
26
26
  )
27
+ pipeline.expire(test_time_key(test_name), config.redis_ttl)
27
28
  end
28
29
  nil
29
30
  end
30
31
 
31
32
  def record_test_name(test_name)
32
- redis.pipelined do
33
- redis.lpush(
33
+ redis.pipelined do |pipeline|
34
+ pipeline.lpush(
34
35
  all_test_names_key,
35
36
  test_name.dup.force_encoding(Encoding::BINARY),
36
37
  )
38
+ pipeline.expire(all_test_names_key, config.redis_ttl)
37
39
  end
38
40
  nil
39
41
  end
40
42
 
41
43
  def fetch_all_test_names
42
- values = redis.pipelined do
43
- redis.lrange(all_test_names_key, 0, -1)
44
+ values = redis.pipelined do |pipeline|
45
+ pipeline.lrange(all_test_names_key, 0, -1)
44
46
  end
45
47
  values.flatten.map(&:to_s)
46
48
  end
47
49
 
48
50
  def fetch_test_time(test_name)
49
- values = redis.pipelined do
50
- key = test_time_key(test_name)
51
- redis.lrange(key, 0, -1)
52
- end
53
- values.flatten.map(&:to_f)
51
+ key = test_time_key(test_name)
52
+ redis.lrange(key, 0, -1).map(&:to_f)
54
53
  end
55
54
 
56
55
  def all_test_names_key
@@ -53,6 +53,10 @@ module CI
53
53
  sleep 0.05
54
54
  end
55
55
  end
56
+ redis.pipelined do |pipeline|
57
+ pipeline.expire(key('worker', worker_id, 'queue'), config.redis_ttl)
58
+ pipeline.expire(key('processed'), config.redis_ttl)
59
+ end
56
60
  rescue *CONNECTION_ERRORS
57
61
  end
58
62
 
@@ -194,13 +198,18 @@ module CI
194
198
  @total = tests.size
195
199
 
196
200
  if @master = redis.setnx(key('master-status'), 'setup')
197
- redis.multi do
198
- redis.lpush(key('queue'), tests) unless tests.empty?
199
- redis.set(key('total'), @total)
200
- redis.set(key('master-status'), 'ready')
201
+ redis.multi do |transaction|
202
+ transaction.lpush(key('queue'), tests) unless tests.empty?
203
+ transaction.set(key('total'), @total)
204
+ transaction.set(key('master-status'), 'ready')
205
+
206
+ transaction.expire(key('queue'), config.redis_ttl)
207
+ transaction.expire(key('total'), config.redis_ttl)
208
+ transaction.expire(key('master-status'), config.redis_ttl)
201
209
  end
202
210
  end
203
211
  register
212
+ redis.expire(key('workers'), config.redis_ttl)
204
213
  rescue *CONNECTION_ERRORS
205
214
  raise if @master
206
215
  end
@@ -2,7 +2,7 @@
2
2
 
3
3
  module CI
4
4
  module Queue
5
- VERSION = '0.22.0'
5
+ VERSION = '0.23.1'
6
6
  DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
7
7
  RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
8
8
  end
@@ -22,6 +22,9 @@ module Minitest
22
22
  def initialize(argv)
23
23
  @queue_config = CI::Queue::Configuration.from_env(ENV)
24
24
  @command, @argv = parse(argv)
25
+ if Minitest.respond_to?(:seed=)
26
+ Minitest.seed = @queue_config.seed.to_i
27
+ end
25
28
  end
26
29
 
27
30
  def run!
@@ -487,6 +490,14 @@ module Minitest
487
490
  end
488
491
  end
489
492
 
493
+ help = <<~EOS
494
+ Defines how long the test report remain after the test run, in seconds.
495
+ Defaults to 28,800 (8 hours)
496
+ EOS
497
+ opts.on("--redis-ttl SECONDS", Integer, help) do |time|
498
+ queue.config.redis_ttl = time
499
+ end
500
+
490
501
  opts.separator ""
491
502
  opts.separator " retry: Replays a previous run in the same order."
492
503
 
data/lib/rspec/queue.rb CHANGED
@@ -157,6 +157,15 @@ module RSpec
157
157
  queue_config.max_consecutive_failures = Integer(max)
158
158
  end
159
159
 
160
+ help = <<~EOS
161
+ Defines how long the test report remain after the test run, in seconds.
162
+ Defaults to 28,800 (8 hours)
163
+ EOS
164
+ parser.separator ""
165
+ parser.on("--redis-ttl SECONDS", Integer, help) do |time|
166
+ queue.config.redis_ttl = time
167
+ end
168
+
160
169
  parser
161
170
  end
162
171
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ci-queue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.22.0
4
+ version: 0.23.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jean Boussier
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-12-14 00:00:00.000000000 Z
11
+ date: 2022-06-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -58,14 +58,14 @@ dependencies:
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: 3.7.0
61
+ version: '3.10'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: 3.7.0
68
+ version: '3.10'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: redis
71
71
  requirement: !ruby/object:Gem::Requirement
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
239
239
  - !ruby/object:Gem::Version
240
240
  version: '0'
241
241
  requirements: []
242
- rubygems_version: 3.2.20
242
+ rubygems_version: 3.3.3
243
243
  signing_key:
244
244
  specification_version: 4
245
245
  summary: Distribute tests over many workers using a queue