gitlab-test-balancing 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/gitlab/test_balancing/client.rb +59 -11
- data/lib/gitlab/test_balancing/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 375d785c08af55c1e66d134855e4410bd97664f628638d32619e33ab837ce669
|
|
4
|
+
data.tar.gz: ebb9818c89abca6fbc0f2e9a74a0941d3734f346e55f2380506f73b7111f9cc9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9ba2af75e6fdcca74d2af6dc0080d7fe0c4b397ad48f29bb01483a614a6b315e1dbda48aa8be0fd5a4852810f795f5d8417c7a451b69a281a9e01946a850d467
|
|
7
|
+
data.tar.gz: 4ec4d7171254f391015124792663fc8235bd56055c67d0a3a8b16f19f881e0283ad65afd398c3f955d38af5d2133cd94b63264ec8d4e4ebc3a8f2c5d9a904d94
|
|
@@ -25,15 +25,29 @@ module Gitlab
|
|
|
25
25
|
ServerError = Class.new(Error)
|
|
26
26
|
private_constant :ServerError
|
|
27
27
|
|
|
28
|
+
# Raised when the API responds 429. Carries the server's Retry-After
|
|
29
|
+
# value (in seconds), if any, so with_retries can honor it.
|
|
30
|
+
TooManyRequestsError = Class.new(Error) do
|
|
31
|
+
attr_reader :retry_after
|
|
32
|
+
|
|
33
|
+
def initialize(message, retry_after: nil)
|
|
34
|
+
super(message)
|
|
35
|
+
@retry_after = retry_after
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
private_constant :TooManyRequestsError
|
|
39
|
+
|
|
28
40
|
Result = Struct.new(:mode, :test_splits, keyword_init: true)
|
|
29
41
|
|
|
30
42
|
OPEN_TIMEOUT = 10
|
|
31
43
|
READ_TIMEOUT = 30
|
|
32
|
-
MAX_ATTEMPTS =
|
|
33
|
-
|
|
44
|
+
MAX_ATTEMPTS = 5
|
|
45
|
+
RETRY_BASE_DELAY = 10
|
|
46
|
+
RETRY_AFTER_MAX_WAIT = 60
|
|
34
47
|
|
|
35
|
-
# Transient errors worth retrying: network failures
|
|
36
|
-
# Other HTTP error responses (e.g. 404/422) are not
|
|
48
|
+
# Transient errors worth retrying: network failures, 5xx responses, and
|
|
49
|
+
# 429 (rate limited). Other HTTP error responses (e.g. 404/422) are not
|
|
50
|
+
# retried.
|
|
37
51
|
RETRIABLE_ERRORS = [
|
|
38
52
|
Net::OpenTimeout,
|
|
39
53
|
Net::ReadTimeout,
|
|
@@ -44,7 +58,8 @@ module Gitlab
|
|
|
44
58
|
SocketError,
|
|
45
59
|
IOError,
|
|
46
60
|
OpenSSL::SSL::SSLError,
|
|
47
|
-
ServerError
|
|
61
|
+
ServerError,
|
|
62
|
+
TooManyRequestsError
|
|
48
63
|
].freeze
|
|
49
64
|
|
|
50
65
|
def initialize(api_url: ENV['CI_API_V4_URL'], job_token: ENV['CI_JOB_TOKEN'], logger: nil)
|
|
@@ -93,6 +108,10 @@ module Gitlab
|
|
|
93
108
|
use_ssl: uri.scheme == 'https', open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT
|
|
94
109
|
) do |http|
|
|
95
110
|
http.request(request).tap do |res|
|
|
111
|
+
if res.is_a?(Net::HTTPTooManyRequests)
|
|
112
|
+
raise TooManyRequestsError.new("#{res.code} #{res.body}", retry_after: retry_after_seconds(res))
|
|
113
|
+
end
|
|
114
|
+
|
|
96
115
|
raise ServerError, "#{res.code} #{res.body}" if res.is_a?(Net::HTTPServerError)
|
|
97
116
|
end
|
|
98
117
|
end
|
|
@@ -108,9 +127,10 @@ module Gitlab
|
|
|
108
127
|
JSON.parse(response.body)
|
|
109
128
|
end
|
|
110
129
|
|
|
111
|
-
# Retry the HTTP request on transient errors with
|
|
112
|
-
# Network failures
|
|
113
|
-
# responses are handled by the caller and are not retried here.
|
|
130
|
+
# Retry the HTTP request on transient errors with backoff and jitter.
|
|
131
|
+
# Network failures, 5xx responses, and 429s are retried; other HTTP
|
|
132
|
+
# error responses are handled by the caller and are not retried here. A
|
|
133
|
+
# 429 additionally honors the server's Retry-After as a wait floor.
|
|
114
134
|
def with_retries(action)
|
|
115
135
|
attempt = 0
|
|
116
136
|
|
|
@@ -118,15 +138,43 @@ module Gitlab
|
|
|
118
138
|
attempt += 1
|
|
119
139
|
yield
|
|
120
140
|
rescue *RETRIABLE_ERRORS => e
|
|
121
|
-
raise Error, "test balancing #{action} failed after #{
|
|
141
|
+
raise Error, "test balancing #{action} failed after #{attempt} attempts: #{e.class}: #{e.message}" if
|
|
122
142
|
attempt >= MAX_ATTEMPTS
|
|
123
143
|
|
|
124
|
-
|
|
125
|
-
|
|
144
|
+
delay = retry_delay(attempt, floor: retry_floor(e))
|
|
145
|
+
logger&.info(
|
|
146
|
+
"test balancing #{action} attempt #{attempt} failed (#{e.class}: #{e.message}), " \
|
|
147
|
+
"retrying in #{delay.round(1)}s..."
|
|
148
|
+
)
|
|
149
|
+
sleep(delay)
|
|
126
150
|
retry
|
|
127
151
|
end
|
|
128
152
|
end
|
|
129
153
|
|
|
154
|
+
# The minimum wait before the next attempt: the server's Retry-After
|
|
155
|
+
# for a 429 (capped, since a misbehaving server could send anything),
|
|
156
|
+
# zero otherwise so those cases fall back to plain backoff.
|
|
157
|
+
def retry_floor(error)
|
|
158
|
+
return 0 unless error.is_a?(TooManyRequestsError) && error.retry_after
|
|
159
|
+
|
|
160
|
+
error.retry_after.clamp(0, RETRY_AFTER_MAX_WAIT)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Full jitter on top of the floor, widening each attempt so nodes blocked
|
|
164
|
+
# in the same rate-limit window spread their retries across the next one.
|
|
165
|
+
def retry_delay(attempt, floor: 0)
|
|
166
|
+
floor + (rand * RETRY_BASE_DELAY * (attempt**1.5))
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# GitLab's Retry-After is always delta-seconds, but parse defensively
|
|
170
|
+
# and skip an unparseable or missing header rather than raise.
|
|
171
|
+
def retry_after_seconds(response)
|
|
172
|
+
value = response['Retry-After']
|
|
173
|
+
return unless value
|
|
174
|
+
|
|
175
|
+
Integer(value, exception: false)
|
|
176
|
+
end
|
|
177
|
+
|
|
130
178
|
def symbolize_test_splits(response)
|
|
131
179
|
Array(response['test_splits']).map { |split| split.transform_keys(&:to_sym) }
|
|
132
180
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab-test-balancing
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- GitLab Engineers
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-11-
|
|
11
|
+
date: 2026-11-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: gitlab-styles
|