ci-queue 0.39.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b107915399e844aa0692d7734747517860631865a6552d4ef3ca4ca8e4be70c5
4
- data.tar.gz: ab35800b2df6a59fec8f6af18e3ab6240cc09818bad1de8fcb23f344541455fc
3
+ metadata.gz: 23effff409f076f80b7af3728d46a4122a59ca1a5a4c3bd1de5162250d60e72f
4
+ data.tar.gz: ae3c402de5003a6acb3ad002305b85c092ae76d4850634892cab935e03d4479b
5
5
  SHA512:
6
- metadata.gz: 53a25b74ed491b2a1d62b8579f2416c7376ba1eaad0831bbb09a45a145177d68e4ac3b86de971e1645cdbabe74d1405109cc325e25932a130099f0996a0058c9
7
- data.tar.gz: 499841246ebd2364397b54a068c033d90b0a5aa670e116678e51002af0eb3c6664bd4585b4e40c841714a2eb969d388707ae8b6a12db60d4ed22269c08c7aa85
6
+ metadata.gz: 0601ef053acbb457931feec6428c627cdc1ef3d8d3e76cdd54a98f02ef6f8f6f531a38be13dc3c40d2800ca0e9efe00c91c21a569470887978e8ba997880933a
7
+ data.tar.gz: c95d7058762b67a059c4868b1d69a8fd99d5f6098d70fa08d71ec7533f7674b8c0886d1090c41db0ba200273b0cbb1c5fbb0fc12ecadddab04f2fb6a9ec950f3
data/dev.yml CHANGED
@@ -3,7 +3,7 @@
3
3
  name: ci-queue
4
4
 
5
5
  up:
6
- - ruby: 2.6.5
6
+ - ruby: 3.3.0
7
7
  - bundler
8
8
  - redis
9
9
 
@@ -5,7 +5,7 @@ module CI
5
5
  attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file
6
6
  attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
7
7
  attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
8
- attr_accessor :max_test_failed, :redis_ttl
8
+ attr_accessor :max_test_failed, :redis_ttl, :warnings_file
9
9
  attr_reader :circuit_breakers
10
10
  attr_writer :seed, :build_id
11
11
  attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -36,7 +36,7 @@ module CI
36
36
  grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
37
37
  max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
38
38
  queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
39
- export_flaky_tests_file: nil
39
+ export_flaky_tests_file: nil, warnings_file: nil
40
40
  )
41
41
  @build_id = build_id
42
42
  @circuit_breakers = [CircuitBreaker::Disabled]
@@ -61,6 +61,7 @@ module CI
61
61
  @report_timeout = report_timeout
62
62
  @inactive_workers_timeout = inactive_workers_timeout
63
63
  @export_flaky_tests_file = export_flaky_tests_file
64
+ @warnings_file = warnings_file
64
65
  end
65
66
 
66
67
  def queue_init_timeout
@@ -13,12 +13,16 @@ module CI
13
13
 
14
14
  def initialize(redis_url, config)
15
15
  @redis_url = redis_url
16
- @redis = ::Redis.new(
17
- url: redis_url,
18
- # Booting a CI worker is costly, so in case of a Redis blip,
19
- # it makes sense to retry for a while before giving up.
20
- reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
21
- )
16
+ if ::Redis::VERSION > "5.0.0"
17
+ @redis = ::Redis.new(
18
+ url: redis_url,
19
+ # Booting a CI worker is costly, so in case of a Redis blip,
20
+ # it makes sense to retry for a while before giving up.
21
+ reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
22
+ )
23
+ else
24
+ @redis = ::Redis.new(url: redis_url)
25
+ end
22
26
  @config = config
23
27
  end
24
28
 
@@ -21,6 +21,13 @@ module CI
21
21
  redis.hkeys(key('error-reports'))
22
22
  end
23
23
 
24
+ TOTAL_KEY = "___total___"
25
+ def requeued_tests
26
+ requeues = redis.hgetall(key('requeues-count'))
27
+ requeues.delete(TOTAL_KEY)
28
+ requeues
29
+ end
30
+
24
31
  def pop_warnings
25
32
  warnings = redis.multi do |transaction|
26
33
  transaction.lrange(key('warnings'), 0, -1)
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- gem 'redis', '>= 5'
4
3
  require 'redis'
5
4
  require 'ci/queue/redis/build_record'
6
5
  require 'ci/queue/redis/base'
@@ -2,7 +2,7 @@
2
2
 
3
3
  module CI
4
4
  module Queue
5
- VERSION = '0.39.0'
5
+ VERSION = '0.41.0'
6
6
  DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
7
7
  RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
8
8
  end
@@ -21,11 +21,21 @@ module Minitest
21
21
  build.flaky_reports
22
22
  end
23
23
 
24
+ def requeued_tests
25
+ build.requeued_tests
26
+ end
27
+
24
28
  def report
25
29
  puts aggregates
26
30
  errors = error_reports
27
31
  puts errors
28
32
 
33
+ requeued_tests.to_a.sort.each do |test_id, count|
34
+ puts yellow("REQUEUE")
35
+ puts "#{test_id} (requeued #{count} times)"
36
+ puts ""
37
+ end
38
+
29
39
  errors.empty?
30
40
  end
31
41
 
@@ -227,6 +227,8 @@ module Minitest
227
227
  end
228
228
 
229
229
  unless supervisor.exhausted?
230
+ reporter = BuildStatusReporter.new(build: supervisor.build)
231
+ reporter.report
230
232
  msg = "#{supervisor.size} tests weren't run."
231
233
  if supervisor.max_test_failed?
232
234
  puts('Encountered too many failed tests. Test run was ended early.')
@@ -296,17 +298,12 @@ module Minitest
296
298
  end
297
299
 
298
300
  def display_warnings(build)
299
- build.pop_warnings.each do |type, attributes|
300
- case type
301
- when CI::Queue::Warnings::RESERVED_LOST_TEST
302
- puts reopen_previous_step
303
- puts yellow(
304
- "[WARNING] #{attributes[:test]} was picked up by another worker because it didn't complete in the allocated #{attributes[:timeout]} seconds.\n" \
305
- "You may want to either optimize this test or bump ci-queue timeout.\n" \
306
- "It's also possible that the worker that was processing it was terminated without being able to report back.\n"
307
- )
308
- end
309
- end
301
+ return unless queue_config.warnings_file
302
+
303
+ warnings = build.pop_warnings.map do |type, attributes|
304
+ attributes.merge(type: type)
305
+ end.compact
306
+ File.write(queue_config.warnings_file, warnings.to_json)
310
307
  end
311
308
 
312
309
  def run_tests_in_fork(queue)
@@ -526,6 +523,15 @@ module Minitest
526
523
  queue_config.export_flaky_tests_file = file
527
524
  end
528
525
 
526
+ help = <<~EOS
527
+ Defines a file where warnings during the execution are written to.
528
+ Defaults to disabled.
529
+ EOS
530
+ opts.separator ""
531
+ opts.on('--warnings-file FILE', help) do |file|
532
+ queue_config.warnings_file = file
533
+ end
534
+
529
535
  help = <<~EOS
530
536
  Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
531
537
  Defaults to disabled.
@@ -244,20 +244,17 @@ module Minitest
244
244
  queue.report_success!
245
245
  end
246
246
 
247
- requeued = false
248
247
  if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
249
- requeued = true
250
248
  result.requeue!
251
249
  reporter.record(result)
252
- elsif queue.acknowledge(example) || !failed
250
+ elsif queue.acknowledge(example)
251
+ reporter.record(result)
252
+ queue.increment_test_failed if failed
253
+ elsif !failed
253
254
  # If the test was already acknowledged by another worker (we timed out)
254
255
  # Then we only record it if it is successful.
255
256
  reporter.record(result)
256
257
  end
257
-
258
- if !requeued && failed
259
- queue.increment_test_failed
260
- end
261
258
  end
262
259
  end
263
260
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ci-queue
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.39.0
4
+ version: 0.41.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jean Boussier
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-12-11 00:00:00.000000000 Z
11
+ date: 2024-01-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
239
239
  - !ruby/object:Gem::Version
240
240
  version: '0'
241
241
  requirements: []
242
- rubygems_version: 3.4.22
242
+ rubygems_version: 3.5.4
243
243
  signing_key:
244
244
  specification_version: 4
245
245
  summary: Distribute tests over many workers using a queue