rspecq 0.2.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +47 -4
- data/Rakefile +1 -1
- data/bin/rspecq +52 -5
- data/lib/rspecq/formatters/failure_recorder.rb +19 -4
- data/lib/rspecq/formatters/job_timing_recorder.rb +3 -0
- data/lib/rspecq/formatters/worker_heartbeat_recorder.rb +0 -1
- data/lib/rspecq/queue.rb +45 -6
- data/lib/rspecq/reporter.rb +24 -11
- data/lib/rspecq/version.rb +1 -1
- data/lib/rspecq/worker.rb +44 -23
- metadata +18 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 86b8792f2ff107cbfb1e67b881724c535df7730804feec7e4813f4690795fac1
|
4
|
+
data.tar.gz: bbd5b78cf7919b653446b00c0748c0ea0d9fca32f69e40e7dd439b3396042dec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 29ee44880ef2d654cd8faeeeb7bcdee2f7ff2b468a22a51a3831d6a175a31246e0cd2206ce012eaa3c1a2ad3aad319360baa10f10d3b84745e7ee9e58fdf3b7b
|
7
|
+
data.tar.gz: d8cd7a9515a25a9b8679acffa99f3b8f19764ff7e23079921677aa40b1941cb6fbce05dbad813f308d68d460f7c06b97a453de2622ee8060161033f65c889189
|
data/CHANGELOG.md
CHANGED
@@ -4,6 +4,44 @@ Breaking changes are prefixed with a "[BREAKING]" label.
|
|
4
4
|
|
5
5
|
## master (unreleased)
|
6
6
|
|
7
|
+
## 0.6.0 (2021-03-23)
|
8
|
+
|
9
|
+
- New cli parameter `seed`.
|
10
|
+
The seed is passed to the RSpec command.
|
11
|
+
|
12
|
+
## 0.5.0 (2021-02-05)
|
13
|
+
|
14
|
+
### Added
|
15
|
+
|
16
|
+
- New cli parameter `queue_wait_timeout`.
|
17
|
+
It configured the time a queue can wait to be ready. The env equivalent
|
18
|
+
is `RSPECQ_QUEUE_WAIT_TIMEOUT`. [#51](https://github.com/skroutz/rspecq/pull/51)
|
19
|
+
|
20
|
+
## 0.4.0 (2020-10-07)
|
21
|
+
|
22
|
+
### Added
|
23
|
+
|
24
|
+
- Builds can be configured to terminate after a specified number of failures,
|
25
|
+
using the `--fail-fast` option.
|
26
|
+
|
27
|
+
|
28
|
+
## 0.3.0 (2020-10-05)
|
29
|
+
|
30
|
+
### Added
|
31
|
+
|
32
|
+
- Providing a Redis URL is now possible using the `--redis-url` option
|
33
|
+
[[#40](https://github.com/skroutz/rspecq/pull/40)]
|
34
|
+
|
35
|
+
### Changed
|
36
|
+
|
37
|
+
- [DEPRECATION] The `--redis` option is now deprecated. Use `--redis-host`
|
38
|
+
instead [[#40](https://github.com/skroutz/rspecq/pull/40)]
|
39
|
+
|
40
|
+
## 0.2.2 (2020-09-10)
|
41
|
+
|
42
|
+
### Fixed
|
43
|
+
- Worker would fail if application code was writing to stderr
|
44
|
+
[[#35](https://github.com/skroutz/rspecq/pull/35)]
|
7
45
|
|
8
46
|
## 0.2.1 (2020-09-09)
|
9
47
|
|
data/README.md
CHANGED
@@ -26,8 +26,8 @@ and [ci-queue](https://github.com/Shopify/ci-queue).
|
|
26
26
|
- Handles intermittent worker failures (e.g. network hiccups, faulty hardware etc.)
|
27
27
|
by detecting non-responsive workers and requeing their jobs. See [*Worker failures*](#worker-failures)
|
28
28
|
- Sentry integration for monitoring build-level events. See [*Sentry integration*](#sentry-integration).
|
29
|
-
- [PLANNED] StatsD integration for various build-level metrics and insights.
|
30
29
|
See [#2](https://github.com/skroutz/rspecq/issues/2).
|
30
|
+
- Automatic termination of builds after a certain amount of failures. See [*Fail-fast*](#fail-fast).
|
31
31
|
|
32
32
|
## Usage
|
33
33
|
|
@@ -64,17 +64,44 @@ USAGE:
|
|
64
64
|
OPTIONS:
|
65
65
|
-b, --build ID A unique identifier for the build. Should be common among workers participating in the same build.
|
66
66
|
-w, --worker ID An identifier for the worker. Workers participating in the same build should have distinct IDs.
|
67
|
-
-r, --redis HOST Redis host to connect to (default: 127.0.0.1).
|
67
|
+
-r, --redis HOST --redis is deprecated. Use --redis-host or --redis-url instead. Redis host to connect to (default: 127.0.0.1).
|
68
|
+
--redis-host HOST Redis host to connect to (default: 127.0.0.1).
|
69
|
+
--redis-url URL Redis URL to connect to (e.g.: redis://127.0.0.1:6379/0).
|
68
70
|
--update-timings Update the global job timings key with the timings of this build. Note: This key is used as the basis for job scheduling.
|
69
71
|
--file-split-threshold N Split spec files slower than N seconds and schedule them as individual examples.
|
70
72
|
--report Enable reporter mode: do not pull tests off the queue; instead print build progress and exit when it's finished.
|
71
73
|
Exits with a non-zero status code if there were any failures.
|
72
74
|
--report-timeout N Fail if build is not finished after N seconds. Only applicable if --report is enabled (default: 3600).
|
73
75
|
--max-requeues N Retry failed examples up to N times before considering them legit failures (default: 3).
|
76
|
+
--queue-wait-timeout N Time to wait for a queue to be ready before considering it failed (default: 30).
|
77
|
+
--fail-fast N Abort build with a non-zero status code after N failed examples.
|
74
78
|
-h, --help Show this message.
|
75
79
|
-v, --version Print the version and exit.
|
76
80
|
```
|
77
81
|
|
82
|
+
You can set most options using ENV variables:
|
83
|
+
|
84
|
+
```shell
|
85
|
+
$ RSPECQ_BUILD=123 RSPECQ_WORKDER=foo1 rspecq spec/
|
86
|
+
```
|
87
|
+
|
88
|
+
### Supported ENV variables
|
89
|
+
|
90
|
+
| Name | Desc |
|
91
|
+
| --- | --- |
|
92
|
+
| `RSPECQ_BUILD` | Build ID |
|
93
|
+
| `RSPECQ_WORKER` | Worker ID |
|
94
|
+
| `RSPECQ_SEED` | RSpec seed |
|
95
|
+
| `RSPECQ_REDIS` | Redis HOST |
|
96
|
+
| `RSPECQ_UPDATE_TIMINGS` | Timings |
|
97
|
+
| `RSPECQ_FILE_SPLIT_THRESHOLD` | File split threshold |
|
98
|
+
| `RSPECQ_REPORT` | Report |
|
99
|
+
| `RSPECQ_REPORT_TIMEOUT` | Report Timeout |
|
100
|
+
| `RSPECQ_MAX_REQUEUES` | Max requests |
|
101
|
+
| `RSPECQ_QUEUE_WAIT_TIMEOUT` | Queue wait timeout |
|
102
|
+
| `RSPECQ_REDIS_URL` | Redis URL |
|
103
|
+
| `RSPECQ_FAIL_FAST` | Fail fast |
|
104
|
+
|
78
105
|
### Sentry integration
|
79
106
|
|
80
107
|
RSpecQ can optionally emit build events to a
|
@@ -86,7 +113,6 @@ This is convenient for monitoring important warnings/errors that may impact
|
|
86
113
|
build times, such as the fact that no previous timings were found and
|
87
114
|
therefore job scheduling was effectively random for a particular build.
|
88
115
|
|
89
|
-
|
90
116
|
## How it works
|
91
117
|
|
92
118
|
The core design is almost identical to ci-queue so please refer to its
|
@@ -113,7 +139,7 @@ For example, a single file may need 10 minutes to run while all other
|
|
113
139
|
files finish after 8 minutes. This would cause all but one workers to be
|
114
140
|
sitting idle for 2 minutes.
|
115
141
|
|
116
|
-
To overcome this issue, RSpecQ can
|
142
|
+
To overcome this issue, RSpecQ can split files which their execution time is
|
117
143
|
above a certain threshold (set with the `--file-split-threshold` option)
|
118
144
|
and instead schedule them as individual examples.
|
119
145
|
|
@@ -131,6 +157,16 @@ final report.
|
|
131
157
|
Flaky tests are also detected and printed as such in the final report. They are
|
132
158
|
also emitted to Sentry (see [Sentry integration](#sentry-integration)).
|
133
159
|
|
160
|
+
### Fail-fast
|
161
|
+
|
162
|
+
In order to prevent large suites running for a long time with a lot of
|
163
|
+
failures, a threshold can be set to control the number of failed examples that
|
164
|
+
will render the build unsuccessful. This is in par with RSpec's
|
165
|
+
[--fail-fast](https://relishapp.com/rspec/rspec-core/docs/command-line/fail-fast-option).
|
166
|
+
|
167
|
+
This feature is disabled by default, and can be controlled via the
|
168
|
+
`--fail-fast` command line option.
|
169
|
+
|
134
170
|
### Worker failures
|
135
171
|
|
136
172
|
It's not uncommon for CI processes to encounter unrecoverable failures for
|
@@ -202,6 +238,13 @@ To enable verbose output in the tests:
|
|
202
238
|
$ RSPECQ_DEBUG=1 bundle exec rake
|
203
239
|
```
|
204
240
|
|
241
|
+
## Redis
|
242
|
+
|
243
|
+
RSpecQ by design doesn't expire its keys from Redis. It is left to the user
|
244
|
+
to configure the Redis server to do so; see
|
245
|
+
[Using Redis as an LRU cache](https://redis.io/topics/lru-cache) for more info.
|
246
|
+
|
247
|
+
You can do this from a configuration file or with `redis-cli`.
|
205
248
|
|
206
249
|
## License
|
207
250
|
|
data/Rakefile
CHANGED
data/bin/rspecq
CHANGED
@@ -2,9 +2,11 @@
|
|
2
2
|
require "optparse"
|
3
3
|
require "rspecq"
|
4
4
|
|
5
|
-
DEFAULT_REDIS_HOST = "127.0.0.1"
|
5
|
+
DEFAULT_REDIS_HOST = "127.0.0.1".freeze
|
6
6
|
DEFAULT_REPORT_TIMEOUT = 3600 # 1 hour
|
7
7
|
DEFAULT_MAX_REQUEUES = 3
|
8
|
+
DEFAULT_QUEUE_WAIT_TIMEOUT = 30
|
9
|
+
DEFAULT_FAIL_FAST = 0
|
8
10
|
|
9
11
|
def env_set?(var)
|
10
12
|
["1", "true"].include?(ENV[var])
|
@@ -36,11 +38,27 @@ OptionParser.new do |o|
|
|
36
38
|
opts[:worker] = v
|
37
39
|
end
|
38
40
|
|
41
|
+
o.on("--seed SEED", "The RSpec seed. Passing the seed can be helpful in " \
|
42
|
+
"many ways i.e reproduction and testing.") do |v|
|
43
|
+
opts[:seed] = v
|
44
|
+
end
|
45
|
+
|
39
46
|
o.on("-r", "--redis HOST", "Redis host to connect to " \
|
40
47
|
"(default: #{DEFAULT_REDIS_HOST}).") do |v|
|
48
|
+
puts "DEPRECATION: --redis is deprecated. Use --redis-host or --redis-url instead"
|
49
|
+
opts[:redis_host] = v
|
50
|
+
end
|
51
|
+
|
52
|
+
o.on("--redis-host HOST", "Redis host to connect to " \
|
53
|
+
"(default: #{DEFAULT_REDIS_HOST}).") do |v|
|
41
54
|
opts[:redis_host] = v
|
42
55
|
end
|
43
56
|
|
57
|
+
o.on("--redis-url URL", "The URL of the Redis host to connect to " \
|
58
|
+
"(e.g.: redis://127.0.0.1:6379/0).") do |v|
|
59
|
+
opts[:redis_url] = v
|
60
|
+
end
|
61
|
+
|
44
62
|
o.on("--update-timings", "Update the global job timings key with the " \
|
45
63
|
"timings of this build. Note: This key is used as the basis for job " \
|
46
64
|
"scheduling.") do |v|
|
@@ -54,7 +72,7 @@ OptionParser.new do |o|
|
|
54
72
|
|
55
73
|
o.on("--report", "Enable reporter mode: do not pull tests off the queue; " \
|
56
74
|
"instead print build progress and exit when it's " \
|
57
|
-
"finished.\n#{o.summary_indent*9} " \
|
75
|
+
"finished.\n#{o.summary_indent * 9} " \
|
58
76
|
"Exits with a non-zero status code if there were any " \
|
59
77
|
"failures.") do |v|
|
60
78
|
opts[:report] = v
|
@@ -72,6 +90,17 @@ OptionParser.new do |o|
|
|
72
90
|
opts[:max_requeues] = v
|
73
91
|
end
|
74
92
|
|
93
|
+
o.on("--queue-wait-timeout N", Integer, "Time to wait for a queue to be " \
|
94
|
+
"ready before considering it failed " \
|
95
|
+
"(default: #{DEFAULT_QUEUE_WAIT_TIMEOUT}).") do |v|
|
96
|
+
opts[:queue_wait_timeout] = v
|
97
|
+
end
|
98
|
+
|
99
|
+
o.on("--fail-fast N", Integer, "Abort build with a non-zero status code " \
|
100
|
+
"after N failed examples.") do |v|
|
101
|
+
opts[:fail_fast] = v
|
102
|
+
end
|
103
|
+
|
75
104
|
o.on_tail("-h", "--help", "Show this message.") do
|
76
105
|
puts o
|
77
106
|
exit
|
@@ -85,21 +114,36 @@ end.parse!
|
|
85
114
|
|
86
115
|
opts[:build] ||= ENV["RSPECQ_BUILD"]
|
87
116
|
opts[:worker] ||= ENV["RSPECQ_WORKER"]
|
117
|
+
opts[:seed] ||= ENV["RSPECQ_SEED"]
|
88
118
|
opts[:redis_host] ||= ENV["RSPECQ_REDIS"] || DEFAULT_REDIS_HOST
|
89
119
|
opts[:timings] ||= env_set?("RSPECQ_UPDATE_TIMINGS")
|
90
|
-
opts[:file_split_threshold] ||= Integer(ENV["RSPECQ_FILE_SPLIT_THRESHOLD"] ||
|
120
|
+
opts[:file_split_threshold] ||= Integer(ENV["RSPECQ_FILE_SPLIT_THRESHOLD"] || 9_999_999)
|
91
121
|
opts[:report] ||= env_set?("RSPECQ_REPORT")
|
92
122
|
opts[:report_timeout] ||= Integer(ENV["RSPECQ_REPORT_TIMEOUT"] || DEFAULT_REPORT_TIMEOUT)
|
93
123
|
opts[:max_requeues] ||= Integer(ENV["RSPECQ_MAX_REQUEUES"] || DEFAULT_MAX_REQUEUES)
|
124
|
+
opts[:queue_wait_timeout] ||= Integer(ENV["RSPECQ_QUEUE_WAIT_TIMEOUT"] || DEFAULT_QUEUE_WAIT_TIMEOUT)
|
125
|
+
opts[:redis_url] ||= ENV["RSPECQ_REDIS_URL"]
|
126
|
+
opts[:fail_fast] ||= Integer(ENV["RSPECQ_FAIL_FAST"] || DEFAULT_FAIL_FAST)
|
94
127
|
|
128
|
+
# rubocop:disable Style/RaiseArgs, Layout/EmptyLineAfterGuardClause
|
95
129
|
raise OptionParser::MissingArgument.new(:build) if opts[:build].nil?
|
96
130
|
raise OptionParser::MissingArgument.new(:worker) if !opts[:report] && opts[:worker].nil?
|
131
|
+
# rubocop:enable Style/RaiseArgs, Layout/EmptyLineAfterGuardClause
|
132
|
+
|
133
|
+
redis_opts = {}
|
134
|
+
|
135
|
+
if opts[:redis_url]
|
136
|
+
redis_opts[:url] = opts[:redis_url]
|
137
|
+
else
|
138
|
+
redis_opts[:host] = opts[:redis_host]
|
139
|
+
end
|
97
140
|
|
98
141
|
if opts[:report]
|
99
142
|
reporter = RSpecQ::Reporter.new(
|
100
143
|
build_id: opts[:build],
|
101
144
|
timeout: opts[:report_timeout],
|
102
|
-
|
145
|
+
redis_opts: redis_opts,
|
146
|
+
queue_wait_timeout: opts[:queue_wait_timeout]
|
103
147
|
)
|
104
148
|
|
105
149
|
reporter.report
|
@@ -107,12 +151,15 @@ else
|
|
107
151
|
worker = RSpecQ::Worker.new(
|
108
152
|
build_id: opts[:build],
|
109
153
|
worker_id: opts[:worker],
|
110
|
-
|
154
|
+
redis_opts: redis_opts
|
111
155
|
)
|
112
156
|
|
113
157
|
worker.files_or_dirs_to_run = ARGV[0] if ARGV[0]
|
114
158
|
worker.populate_timings = opts[:timings]
|
115
159
|
worker.file_split_threshold = opts[:file_split_threshold]
|
116
160
|
worker.max_requeues = opts[:max_requeues]
|
161
|
+
worker.queue_wait_timeout = opts[:queue_wait_timeout]
|
162
|
+
worker.fail_fast = opts[:fail_fast]
|
163
|
+
worker.seed = Integer(opts[:seed]) if opts[:seed]
|
117
164
|
worker.work
|
118
165
|
end
|
@@ -1,5 +1,10 @@
|
|
1
1
|
module RSpecQ
|
2
2
|
module Formatters
|
3
|
+
# Persists failed examples information (i.e. message and backtrace), so
|
4
|
+
# that they can be reported to the end user by the Reporter.
|
5
|
+
#
|
6
|
+
# Also persists non-example error information (e.g. a syntax error that
|
7
|
+
# in a spec file).
|
3
8
|
class FailureRecorder
|
4
9
|
def initialize(queue, job, max_requeues)
|
5
10
|
@queue = queue
|
@@ -25,7 +30,14 @@ module RSpecQ
|
|
25
30
|
def example_failed(notification)
|
26
31
|
example = notification.example
|
27
32
|
|
33
|
+
rerun_cmd = "bin/rspec --seed #{RSpec.configuration.seed} #{example.location_rerun_argument}"
|
34
|
+
|
28
35
|
if @queue.requeue_job(example.id, @max_requeues)
|
36
|
+
|
37
|
+
# Save the rerun command for later. It will be used if this is
|
38
|
+
# a flaky test for more user-friendly reporting.
|
39
|
+
@queue.save_rerun_command(example.id, rerun_cmd)
|
40
|
+
|
29
41
|
# HACK: try to avoid picking the job we just requeued; we want it
|
30
42
|
# to be picked up by a different worker
|
31
43
|
sleep 0.5
|
@@ -33,16 +45,19 @@ module RSpecQ
|
|
33
45
|
end
|
34
46
|
|
35
47
|
presenter = RSpec::Core::Formatters::ExceptionPresenter.new(
|
36
|
-
example.exception, example
|
48
|
+
example.exception, example
|
49
|
+
)
|
37
50
|
|
38
51
|
msg = presenter.fully_formatted(nil, @colorizer)
|
39
52
|
msg << "\n"
|
40
53
|
msg << @colorizer.wrap(
|
41
|
-
|
42
|
-
RSpec.configuration.failure_color
|
54
|
+
rerun_cmd,
|
55
|
+
RSpec.configuration.failure_color
|
56
|
+
)
|
43
57
|
|
44
58
|
msg << @colorizer.wrap(
|
45
|
-
" # #{example.full_description}", RSpec.configuration.detail_color
|
59
|
+
" # #{example.full_description}", RSpec.configuration.detail_color
|
60
|
+
)
|
46
61
|
|
47
62
|
@queue.record_example_failure(notification.example.id, msg)
|
48
63
|
end
|
@@ -1,5 +1,8 @@
|
|
1
1
|
module RSpecQ
|
2
2
|
module Formatters
|
3
|
+
# Persists each job's timing (in seconds). Those timings are used when
|
4
|
+
# determining the ordering in which jobs are scheduled (slower jobs will
|
5
|
+
# be enqueued first).
|
3
6
|
class JobTimingRecorder
|
4
7
|
def initialize(queue, job)
|
5
8
|
@queue = queue
|
data/lib/rspecq/queue.rb
CHANGED
@@ -70,15 +70,16 @@ module RSpecQ
|
|
70
70
|
|
71
71
|
attr_reader :redis
|
72
72
|
|
73
|
-
def initialize(build_id, worker_id,
|
73
|
+
def initialize(build_id, worker_id, redis_opts)
|
74
74
|
@build_id = build_id
|
75
75
|
@worker_id = worker_id
|
76
|
-
@redis = Redis.new(
|
76
|
+
@redis = Redis.new(redis_opts.merge(id: worker_id))
|
77
77
|
end
|
78
78
|
|
79
79
|
# NOTE: jobs will be processed from head to tail (lpop)
|
80
|
-
def publish(jobs)
|
80
|
+
def publish(jobs, fail_fast = 0)
|
81
81
|
@redis.multi do
|
82
|
+
@redis.hset(key_queue_config, "fail_fast", fail_fast)
|
82
83
|
@redis.rpush(key_queue_unprocessed, jobs)
|
83
84
|
@redis.set(key_queue_status, STATUS_READY)
|
84
85
|
end.first
|
@@ -130,10 +131,18 @@ module RSpecQ
|
|
130
131
|
@redis.eval(
|
131
132
|
REQUEUE_JOB,
|
132
133
|
keys: [key_queue_unprocessed, key_requeues],
|
133
|
-
argv: [job, max_requeues]
|
134
|
+
argv: [job, max_requeues]
|
134
135
|
)
|
135
136
|
end
|
136
137
|
|
138
|
+
def save_rerun_command(job, cmd)
|
139
|
+
@redis.hset(key("job_metadata"), job, cmd)
|
140
|
+
end
|
141
|
+
|
142
|
+
def rerun_command(job)
|
143
|
+
@redis.hget(key("job_metadata"), job)
|
144
|
+
end
|
145
|
+
|
137
146
|
def record_example_failure(example_id, message)
|
138
147
|
@redis.hset(key_failures, example_id, message)
|
139
148
|
end
|
@@ -209,9 +218,10 @@ module RSpecQ
|
|
209
218
|
@redis.get(key_queue_status) == STATUS_READY
|
210
219
|
end
|
211
220
|
|
212
|
-
def wait_until_published(timeout=30)
|
221
|
+
def wait_until_published(timeout = 30)
|
213
222
|
(timeout * 10).times do
|
214
223
|
return if published?
|
224
|
+
|
215
225
|
sleep 0.1
|
216
226
|
end
|
217
227
|
|
@@ -232,7 +242,9 @@ module RSpecQ
|
|
232
242
|
# after being retried). Must be called after the build is complete,
|
233
243
|
# otherwise an exception will be raised.
|
234
244
|
def flaky_jobs
|
235
|
-
|
245
|
+
if !exhausted? && !build_failed_fast?
|
246
|
+
raise "Queue is not yet exhausted"
|
247
|
+
end
|
236
248
|
|
237
249
|
requeued = @redis.hkeys(key_requeues)
|
238
250
|
|
@@ -241,11 +253,38 @@ module RSpecQ
|
|
241
253
|
requeued - @redis.hkeys(key_failures)
|
242
254
|
end
|
243
255
|
|
256
|
+
# Returns the number of failures that will trigger the build to fail-fast.
|
257
|
+
# Returns 0 if this feature is disabled and nil if the Queue is not yet
|
258
|
+
# published
|
259
|
+
def fail_fast
|
260
|
+
return nil unless published?
|
261
|
+
|
262
|
+
@fail_fast ||= Integer(@redis.hget(key_queue_config, "fail_fast"))
|
263
|
+
end
|
264
|
+
|
265
|
+
# Returns true if the number of failed tests, has surpassed the threshold
|
266
|
+
# to render the run unsuccessful and the build should be terminated.
|
267
|
+
def build_failed_fast?
|
268
|
+
if fail_fast.nil? || fail_fast.zero?
|
269
|
+
return false
|
270
|
+
end
|
271
|
+
|
272
|
+
@redis.multi do
|
273
|
+
@redis.hlen(key_failures)
|
274
|
+
@redis.hlen(key_errors)
|
275
|
+
end.inject(:+) >= fail_fast
|
276
|
+
end
|
277
|
+
|
244
278
|
# redis: STRING [STATUS_INITIALIZING, STATUS_READY]
|
245
279
|
def key_queue_status
|
246
280
|
key("queue", "status")
|
247
281
|
end
|
248
282
|
|
283
|
+
# redis: HASH<config_key => config_value>
|
284
|
+
def key_queue_config
|
285
|
+
key("queue", "config")
|
286
|
+
end
|
287
|
+
|
249
288
|
# redis: LIST<job>
|
250
289
|
def key_queue_unprocessed
|
251
290
|
key("queue", "unprocessed")
|
data/lib/rspecq/reporter.rb
CHANGED
@@ -9,18 +9,19 @@ module RSpecQ
|
|
9
9
|
#
|
10
10
|
# Reporters are readers of the queue.
|
11
11
|
class Reporter
|
12
|
-
def initialize(build_id:, timeout:,
|
12
|
+
def initialize(build_id:, timeout:, redis_opts:, queue_wait_timeout: 30)
|
13
13
|
@build_id = build_id
|
14
14
|
@timeout = timeout
|
15
|
-
@queue = Queue.new(build_id, "reporter",
|
15
|
+
@queue = Queue.new(build_id, "reporter", redis_opts)
|
16
|
+
@queue_wait_timeout = queue_wait_timeout
|
16
17
|
|
17
18
|
# We want feedback to be immediattely printed to CI users, so
|
18
19
|
# we disable buffering.
|
19
|
-
|
20
|
+
$stdout.sync = true
|
20
21
|
end
|
21
22
|
|
22
23
|
def report
|
23
|
-
@queue.wait_until_published
|
24
|
+
@queue.wait_until_published(@queue_wait_timeout)
|
24
25
|
|
25
26
|
finished = false
|
26
27
|
|
@@ -28,7 +29,7 @@ module RSpecQ
|
|
28
29
|
failure_heading_printed = false
|
29
30
|
|
30
31
|
tests_duration = measure_duration do
|
31
|
-
@timeout.times do
|
32
|
+
@timeout.times do
|
32
33
|
@queue.example_failures.each do |job, rspec_output|
|
33
34
|
next if reported_failures[job]
|
34
35
|
|
@@ -41,7 +42,7 @@ module RSpecQ
|
|
41
42
|
puts failure_formatted(rspec_output)
|
42
43
|
end
|
43
44
|
|
44
|
-
|
45
|
+
unless @queue.exhausted? || @queue.build_failed_fast?
|
45
46
|
sleep 1
|
46
47
|
next
|
47
48
|
end
|
@@ -55,7 +56,7 @@ module RSpecQ
|
|
55
56
|
|
56
57
|
@queue.record_build_time(tests_duration)
|
57
58
|
|
58
|
-
flaky_jobs = @queue.flaky_jobs
|
59
|
+
flaky_jobs = @queue.flaky_jobs.map { |job| @queue.rerun_command(job) }
|
59
60
|
|
60
61
|
puts summary(@queue.example_failures, @queue.non_example_errors,
|
61
62
|
flaky_jobs, humanize_duration(tests_duration))
|
@@ -83,6 +84,13 @@ module RSpecQ
|
|
83
84
|
end
|
84
85
|
|
85
86
|
summary = ""
|
87
|
+
if @queue.build_failed_fast?
|
88
|
+
summary << "\n\n"
|
89
|
+
summary << "The limit of #{@queue.fail_fast} failures has been reached\n"
|
90
|
+
summary << "Aborting..."
|
91
|
+
summary << "\n"
|
92
|
+
end
|
93
|
+
|
86
94
|
summary << failed_examples_section if !failures.empty?
|
87
95
|
|
88
96
|
errors.each { |_job, msg| summary << msg }
|
@@ -99,7 +107,12 @@ module RSpecQ
|
|
99
107
|
if !flaky_jobs.empty?
|
100
108
|
summary << "\n\n"
|
101
109
|
summary << "Flaky jobs detected (count=#{flaky_jobs.count}):\n"
|
102
|
-
flaky_jobs.each
|
110
|
+
flaky_jobs.each do |j|
|
111
|
+
summary << RSpec::Core::Formatters::ConsoleCodes.wrap(
|
112
|
+
"#{j}\n",
|
113
|
+
RSpec.configuration.pending_color
|
114
|
+
)
|
115
|
+
end
|
103
116
|
end
|
104
117
|
|
105
118
|
summary
|
@@ -117,13 +130,13 @@ module RSpecQ
|
|
117
130
|
return if jobs.empty?
|
118
131
|
|
119
132
|
jobs.each do |job|
|
120
|
-
filename = job.sub(/\[.+\]/,
|
133
|
+
filename = job.sub(/\[.+\]/, "")
|
121
134
|
|
122
135
|
extra = {
|
123
136
|
build: @build_id,
|
124
137
|
build_timeout: @timeout,
|
125
138
|
queue: @queue.inspect,
|
126
|
-
object:
|
139
|
+
object: inspect,
|
127
140
|
pid: Process.pid,
|
128
141
|
job_path: job,
|
129
142
|
build_duration: build_duration
|
@@ -136,7 +149,7 @@ module RSpecQ
|
|
136
149
|
|
137
150
|
Raven.capture_message(
|
138
151
|
"Flaky test in #{filename}",
|
139
|
-
level:
|
152
|
+
level: "warning",
|
140
153
|
extra: extra,
|
141
154
|
tags: tags
|
142
155
|
)
|
data/lib/rspecq/version.rb
CHANGED
data/lib/rspecq/worker.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require "json"
|
2
2
|
require "pathname"
|
3
3
|
require "pp"
|
4
|
+
require "open3"
|
4
5
|
|
5
6
|
module RSpecQ
|
6
7
|
# A Worker, given a build ID, continuously consumes tests off the
|
@@ -39,17 +40,34 @@ module RSpecQ
|
|
39
40
|
# Defaults to 3
|
40
41
|
attr_accessor :max_requeues
|
41
42
|
|
43
|
+
# Stop the execution after N failed tests. Do not stop at any point when
|
44
|
+
# set to 0.
|
45
|
+
#
|
46
|
+
# Defaults to 0
|
47
|
+
attr_accessor :fail_fast
|
48
|
+
|
49
|
+
# Time to wait for a queue to be published.
|
50
|
+
#
|
51
|
+
# Defaults to 30
|
52
|
+
attr_accessor :queue_wait_timeout
|
53
|
+
|
54
|
+
# The RSpec seed
|
55
|
+
attr_accessor :seed
|
56
|
+
|
42
57
|
attr_reader :queue
|
43
58
|
|
44
|
-
def initialize(build_id:, worker_id:,
|
59
|
+
def initialize(build_id:, worker_id:, redis_opts:)
|
45
60
|
@build_id = build_id
|
46
61
|
@worker_id = worker_id
|
47
|
-
@queue = Queue.new(build_id, worker_id,
|
62
|
+
@queue = Queue.new(build_id, worker_id, redis_opts)
|
63
|
+
@fail_fast = 0
|
48
64
|
@files_or_dirs_to_run = "spec"
|
49
65
|
@populate_timings = false
|
50
|
-
@file_split_threshold =
|
66
|
+
@file_split_threshold = 999_999
|
51
67
|
@heartbeat_updated_at = nil
|
52
68
|
@max_requeues = 3
|
69
|
+
@queue_wait_timeout = 30
|
70
|
+
@seed = srand && srand % 0xFFFF
|
53
71
|
|
54
72
|
RSpec::Core::Formatters.register(Formatters::JobTimingRecorder, :dump_summary)
|
55
73
|
RSpec::Core::Formatters.register(Formatters::ExampleCountRecorder, :dump_summary)
|
@@ -61,13 +79,15 @@ module RSpecQ
|
|
61
79
|
puts "Working for build #{@build_id} (worker=#{@worker_id})"
|
62
80
|
|
63
81
|
try_publish_queue!(queue)
|
64
|
-
queue.wait_until_published
|
82
|
+
queue.wait_until_published(queue_wait_timeout)
|
65
83
|
|
66
84
|
loop do
|
67
85
|
# we have to bootstrap this so that it can be used in the first call
|
68
86
|
# to `requeue_lost_job` inside the work loop
|
69
87
|
update_heartbeat
|
70
88
|
|
89
|
+
return if queue.build_failed_fast?
|
90
|
+
|
71
91
|
lost = queue.requeue_lost_job
|
72
92
|
puts "Requeued lost job: #{lost}" if lost
|
73
93
|
|
@@ -87,8 +107,8 @@ module RSpecQ
|
|
87
107
|
|
88
108
|
# reconfigure rspec
|
89
109
|
RSpec.configuration.detail_color = :magenta
|
90
|
-
RSpec.configuration.seed =
|
91
|
-
RSpec.configuration.backtrace_formatter.filter_gem(
|
110
|
+
RSpec.configuration.seed = seed
|
111
|
+
RSpec.configuration.backtrace_formatter.filter_gem("rspecq")
|
92
112
|
RSpec.configuration.add_formatter(Formatters::FailureRecorder.new(queue, job, max_requeues))
|
93
113
|
RSpec.configuration.add_formatter(Formatters::ExampleCountRecorder.new(queue))
|
94
114
|
RSpec.configuration.add_formatter(Formatters::WorkerHeartbeatRecorder.new(self))
|
@@ -120,7 +140,7 @@ module RSpecQ
|
|
120
140
|
|
121
141
|
timings = queue.timings
|
122
142
|
if timings.empty?
|
123
|
-
q_size = queue.publish(files_to_run.shuffle)
|
143
|
+
q_size = queue.publish(files_to_run.shuffle, fail_fast)
|
124
144
|
log_event(
|
125
145
|
"No timings found! Published queue in random order (size=#{q_size})",
|
126
146
|
"warning"
|
@@ -145,7 +165,7 @@ module RSpecQ
|
|
145
165
|
jobs.concat(files_to_run)
|
146
166
|
end
|
147
167
|
|
148
|
-
default_timing = timings.values[timings.values.size/2]
|
168
|
+
default_timing = timings.values[timings.values.size / 2]
|
149
169
|
|
150
170
|
# assign timings (based on previous runs) to all jobs
|
151
171
|
jobs = jobs.each_with_object({}) do |j, h|
|
@@ -159,7 +179,7 @@ module RSpecQ
|
|
159
179
|
# sort jobs based on their timings (slowest to be processed first)
|
160
180
|
jobs = jobs.sort_by { |_j, t| -t }.map(&:first)
|
161
181
|
|
162
|
-
puts "Published queue (size=#{queue.publish(jobs)})"
|
182
|
+
puts "Published queue (size=#{queue.publish(jobs, fail_fast)})"
|
163
183
|
end
|
164
184
|
|
165
185
|
private
|
@@ -170,7 +190,8 @@ module RSpecQ
|
|
170
190
|
# see https://github.com/rspec/rspec-core/pull/2723
|
171
191
|
if Gem::Version.new(RSpec::Core::Version::STRING) <= Gem::Version.new("3.9.1")
|
172
192
|
RSpec.world.instance_variable_set(
|
173
|
-
:@example_group_counts_by_spec_file, Hash.new(0)
|
193
|
+
:@example_group_counts_by_spec_file, Hash.new(0)
|
194
|
+
)
|
174
195
|
end
|
175
196
|
|
176
197
|
# RSpec.clear_examples does not reset those, which causes issues when
|
@@ -189,22 +210,22 @@ module RSpecQ
|
|
189
210
|
# falling back to scheduling them as whole files. Their errors will be
|
190
211
|
# reported in the normal flow when they're eventually picked up by a worker.
|
191
212
|
def files_to_example_ids(files)
|
192
|
-
cmd = "DISABLE_SPRING=1 bundle exec rspec --dry-run --format json #{files.join(' ')}
|
193
|
-
out =
|
194
|
-
cmd_result = $?
|
213
|
+
cmd = "DISABLE_SPRING=1 bundle exec rspec --dry-run --format json #{files.join(' ')}"
|
214
|
+
out, err, cmd_result = Open3.capture3(cmd)
|
195
215
|
|
196
216
|
if !cmd_result.success?
|
197
217
|
rspec_output = begin
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
218
|
+
JSON.parse(out)
|
219
|
+
rescue JSON::ParserError
|
220
|
+
out
|
221
|
+
end
|
202
222
|
|
203
223
|
log_event(
|
204
|
-
"Failed to split slow files, falling back to regular scheduling",
|
224
|
+
"Failed to split slow files, falling back to regular scheduling.\n #{err}",
|
205
225
|
"error",
|
206
|
-
|
207
|
-
|
226
|
+
rspec_stdout: rspec_output,
|
227
|
+
rspec_stderr: err,
|
228
|
+
cmd_result: cmd_result.inspect
|
208
229
|
)
|
209
230
|
|
210
231
|
pp rspec_output
|
@@ -226,7 +247,7 @@ module RSpecQ
|
|
226
247
|
|
227
248
|
# Prints msg to standard output and emits an event to Sentry, if the
|
228
249
|
# SENTRY_DSN environment variable is set.
|
229
|
-
def log_event(msg, level, additional={})
|
250
|
+
def log_event(msg, level, additional = {})
|
230
251
|
puts msg
|
231
252
|
|
232
253
|
Raven.capture_message(msg, level: level, extra: {
|
@@ -237,8 +258,8 @@ module RSpecQ
|
|
237
258
|
populate_timings: populate_timings,
|
238
259
|
file_split_threshold: file_split_threshold,
|
239
260
|
heartbeat_updated_at: @heartbeat_updated_at,
|
240
|
-
object:
|
241
|
-
pid: Process.pid
|
261
|
+
object: inspect,
|
262
|
+
pid: Process.pid
|
242
263
|
}.merge(additional))
|
243
264
|
end
|
244
265
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rspecq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Agis Anastasopoulos
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec-core
|
@@ -53,7 +53,7 @@ dependencies:
|
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: minitest
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - ">="
|
@@ -81,7 +81,7 @@ dependencies:
|
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
84
|
+
name: rake
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - ">="
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rubocop
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.93.0
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 0.93.0
|
111
125
|
description:
|
112
126
|
email: agis.anast@gmail.com
|
113
127
|
executables:
|