gitlab-labkit 1.22.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.copier-answers.yml +2 -1
- data/.gitlab-ci-other-versions.yml +5 -0
- data/.gitlab-ci.yml +1 -1
- data/docker-compose.yml +1 -1
- data/gitlab-labkit.gemspec +7 -7
- data/lib/gitlab-labkit.rb +1 -0
- data/lib/labkit/rate_limit/evaluator.rb +130 -18
- data/lib/labkit/rate_limit/limiter.rb +5 -2
- data/lib/labkit/rate_limit/result.rb +10 -5
- data/lib/labkit/rate_limit/rule.rb +26 -3
- data/lib/labkit/rate_limit.rb +3 -2
- data/lib/labkit/redis/script.rb +43 -0
- data/lib/labkit/redis.rb +8 -0
- metadata +46 -19
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1457da9fa7cec587b47834870a4eb60f5a21c9966111d1d9edd0e7e0a49b4ff3
|
|
4
|
+
data.tar.gz: 631e56481d2be50e976a16aefa6fd6a250759089b56dde0414af059fd55c4ba7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5d6aba7952cc495dd4b74ea0aa452f473e01dc6a6bda2a94d1238d9102f5b448de2b4ae1765f87e2020d2fb460b1613044fe9403f0019b54b050b3a48261b425
|
|
7
|
+
data.tar.gz: 21ddb1d302b53049a605052d67bbdf2d51bee9a4e80f82c6c10f689d5b292397e0d913dae4b05e66c93ee4c93d831e14e885c448a77b049dab48ac1bda3b84b5
|
data/.copier-answers.yml
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# See the project for instructions on how to update the project
|
|
4
4
|
#
|
|
5
5
|
# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY
|
|
6
|
-
_commit: v1.
|
|
6
|
+
_commit: v1.50.0
|
|
7
7
|
_src_path: https://gitlab.com/gitlab-com/gl-infra/common-template-copier.git
|
|
8
8
|
ee_licensed: false
|
|
9
9
|
gitlab_namespace: gitlab-org/ruby/gems
|
|
@@ -11,6 +11,7 @@ golang: false
|
|
|
11
11
|
helm: false
|
|
12
12
|
initial_codeowners: '@reprazent @andrewn @mkaeppler @ayufan'
|
|
13
13
|
jsonnet: false
|
|
14
|
+
measure_performance: false
|
|
14
15
|
project_name: labkit-ruby
|
|
15
16
|
release_platform: false
|
|
16
17
|
ruby: true
|
data/.gitlab-ci.yml
CHANGED
data/docker-compose.yml
CHANGED
data/gitlab-labkit.gemspec
CHANGED
|
@@ -22,17 +22,17 @@ Gem::Specification.new do |spec|
|
|
|
22
22
|
spec.required_ruby_version = ">= 3.3", "< 5"
|
|
23
23
|
|
|
24
24
|
# Please maintain alphabetical order for dependencies
|
|
25
|
-
spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 8.
|
|
26
|
-
spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 8.
|
|
25
|
+
spec.add_runtime_dependency "actionpack", ">= 5.0.0", "< 8.2.0"
|
|
26
|
+
spec.add_runtime_dependency "activesupport", ">= 5.0.0", "< 8.2.0"
|
|
27
27
|
spec.add_runtime_dependency "grpc", ">= 1.75" # Be sure to update the "grpc-tools" dev_dependency too
|
|
28
28
|
spec.add_runtime_dependency "google-protobuf", ">= 3.25", "< 5.0"
|
|
29
29
|
spec.add_runtime_dependency "jaeger-client", "~> 1.1.0"
|
|
30
30
|
spec.add_runtime_dependency "json_schemer", ">= 2.3.0", "< 3.0"
|
|
31
31
|
spec.add_runtime_dependency "openssl", "~> 3.3.2"
|
|
32
|
-
spec.add_runtime_dependency "opentelemetry-sdk", "
|
|
33
|
-
spec.add_runtime_dependency "opentelemetry-instrumentation-all", "
|
|
34
|
-
spec.add_runtime_dependency "opentelemetry-exporter-otlp", "
|
|
35
|
-
spec.add_runtime_dependency "opentracing", "
|
|
32
|
+
spec.add_runtime_dependency "opentelemetry-sdk", ">= 1.10", "< 2"
|
|
33
|
+
spec.add_runtime_dependency "opentelemetry-instrumentation-all", ">= 0.89", "< 1"
|
|
34
|
+
spec.add_runtime_dependency "opentelemetry-exporter-otlp", ">= 0.31", "< 1"
|
|
35
|
+
spec.add_runtime_dependency "opentracing", ">= 0.4", "< 1"
|
|
36
36
|
spec.add_runtime_dependency "pg_query", ">= 6.1.0", "< 7.0"
|
|
37
37
|
spec.add_runtime_dependency "prometheus-client-mmap", ">= 1.2", "< 2.0"
|
|
38
38
|
spec.add_runtime_dependency "redis", "> 3.0.0", "< 6.0.0"
|
|
@@ -49,7 +49,7 @@ Gem::Specification.new do |spec|
|
|
|
49
49
|
spec.add_development_dependency "pry", "~> 0.12"
|
|
50
50
|
spec.add_development_dependency "pry-byebug", "~> 3.11"
|
|
51
51
|
spec.add_development_dependency "rack", "~> 2.0"
|
|
52
|
-
spec.add_development_dependency "railties", ">= 5.0.0", "< 8.
|
|
52
|
+
spec.add_development_dependency "railties", ">= 5.0.0", "< 8.2.0"
|
|
53
53
|
spec.add_development_dependency "rake", "~> 13.2"
|
|
54
54
|
spec.add_development_dependency "rest-client", "~> 2.1.0"
|
|
55
55
|
spec.add_development_dependency "rspec", "~> 3.12.0"
|
data/lib/gitlab-labkit.rb
CHANGED
|
@@ -25,6 +25,7 @@ module Labkit
|
|
|
25
25
|
autoload :Middleware, "labkit/middleware"
|
|
26
26
|
autoload :Fields, "labkit/fields"
|
|
27
27
|
autoload :RateLimit, "labkit/rate_limit"
|
|
28
|
+
autoload :Redis, "labkit/redis"
|
|
28
29
|
|
|
29
30
|
# Publishers to publish notifications whenever a HTTP reqeust is made.
|
|
30
31
|
# A broadcasted notification's payload in topic "request.external_http" includes:
|
|
@@ -12,6 +12,52 @@ module Labkit
|
|
|
12
12
|
CHAR_VALUE_MAX_LENGTH = 200
|
|
13
13
|
MISSING_VALUE_SENTINEL = "_unknown_"
|
|
14
14
|
|
|
15
|
+
# Atomic increment-with-TTL Lua script. The whole script runs as one
|
|
16
|
+
# operation from Redis's perspective, so there is no window between
|
|
17
|
+
# the increment and EXPIRE that can leak a key without TTL.
|
|
18
|
+
#
|
|
19
|
+
# INCRBYFLOAT serves both count-mode (cost=1, equivalent to INCR for
|
|
20
|
+
# integer-encoded keys) and cost-mode callers, so a single script
|
|
21
|
+
# handles every rule shape. cost=0 also flows through INCRBYFLOAT;
|
|
22
|
+
# Redis treats the result as a no-op on the stored value while
|
|
23
|
+
# still observing the post-state count and TTL we return.
|
|
24
|
+
#
|
|
25
|
+
# ttl_before < 0 covers TTL=-2 (key missing) and TTL=-1 (no expiry).
|
|
26
|
+
# The -1 case shouldn't arise with the atomic script, but self-healing
|
|
27
|
+
# recovers keys left without TTL by any prior bug.
|
|
28
|
+
INCR_SCRIPT = Labkit::Redis::Script.new(<<~LUA)
|
|
29
|
+
local ttl = ARGV[1]
|
|
30
|
+
local cost = tonumber(ARGV[2])
|
|
31
|
+
local ttl_before = redis.call('TTL', KEYS[1])
|
|
32
|
+
|
|
33
|
+
local count = redis.call('INCRBYFLOAT', KEYS[1], cost)
|
|
34
|
+
if ttl_before < 0 then
|
|
35
|
+
redis.call('EXPIRE', KEYS[1], ttl)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
return {count, redis.call('TTL', KEYS[1])}
|
|
39
|
+
LUA
|
|
40
|
+
|
|
41
|
+
# Atomic SADD + SCARD + conditional EXPIRE. SET-cardinality counterpart
|
|
42
|
+
# of INCR_SCRIPT; same shape (read TTL, mutate, set TTL when missing,
|
|
43
|
+
# return post-state {count, TTL}). count is SCARD, not the SADD return.
|
|
44
|
+
#
|
|
45
|
+
# ttl_before < 0 covers TTL=-2 (key missing) and TTL=-1 (no expiry),
|
|
46
|
+
# so this also self-heals orphan keys left without TTL.
|
|
47
|
+
SADD_SCRIPT = Labkit::Redis::Script.new(<<~LUA)
|
|
48
|
+
local ttl = ARGV[1]
|
|
49
|
+
local member = ARGV[2]
|
|
50
|
+
local ttl_before = redis.call('TTL', KEYS[1])
|
|
51
|
+
|
|
52
|
+
redis.call('SADD', KEYS[1], member)
|
|
53
|
+
local count = redis.call('SCARD', KEYS[1])
|
|
54
|
+
if ttl_before < 0 then
|
|
55
|
+
redis.call('EXPIRE', KEYS[1], ttl)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
return {count, redis.call('TTL', KEYS[1])}
|
|
59
|
+
LUA
|
|
60
|
+
|
|
15
61
|
def initialize(name:, rules:, redis:, logger:)
|
|
16
62
|
@name = name
|
|
17
63
|
@rules = rules
|
|
@@ -19,8 +65,8 @@ module Labkit
|
|
|
19
65
|
@logger = logger
|
|
20
66
|
end
|
|
21
67
|
|
|
22
|
-
def check(identifier)
|
|
23
|
-
check_rules(identifier)
|
|
68
|
+
def check(identifier, cost: 1)
|
|
69
|
+
check_rules(identifier, cost)
|
|
24
70
|
rescue StandardError => e
|
|
25
71
|
# Intentionally broad: fail-open applies to any unexpected error (network,
|
|
26
72
|
# timeout, OOM) not only Redis protocol errors.
|
|
@@ -44,11 +90,22 @@ module Labkit
|
|
|
44
90
|
|
|
45
91
|
# :log rules are non-terminating: they emit metrics and continue,
|
|
46
92
|
# so a shadow :log rule cannot disable a following :block rule.
|
|
47
|
-
|
|
93
|
+
#
|
|
94
|
+
# SET-mode rules (rule.count_distinct set) that match but whose identifier
|
|
95
|
+
# is missing the count_distinct key fail open + log + bump errors_total, and
|
|
96
|
+
# the loop continues to the next rule (the rule is treated as not applicable
|
|
97
|
+
# rather than aborting the whole evaluation).
|
|
98
|
+
def check_rules(identifier, cost)
|
|
48
99
|
@rules.each do |rule|
|
|
49
100
|
next unless rule_matches?(rule, identifier)
|
|
50
101
|
|
|
51
|
-
|
|
102
|
+
if rule.count_distinct && missing_count_distinct_value?(rule, identifier)
|
|
103
|
+
log_missing_count_distinct(rule, identifier)
|
|
104
|
+
report_error_metrics
|
|
105
|
+
next
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
result = evaluate_rule(rule, identifier, cost)
|
|
52
109
|
report_matched_metrics(result)
|
|
53
110
|
return result unless rule.action == :log
|
|
54
111
|
end
|
|
@@ -59,6 +116,10 @@ module Labkit
|
|
|
59
116
|
|
|
60
117
|
# Mirror of check_rules without metrics: peek skips :log rules (their state
|
|
61
118
|
# is unobservable through peek).
|
|
119
|
+
#
|
|
120
|
+
# peek does not need the count_distinct identifier key - it reads SCARD on
|
|
121
|
+
# the rule-keyed compound key, which contains the cardinality across all
|
|
122
|
+
# members. So missing-key fail-open does not apply here.
|
|
62
123
|
def peek_rules(identifier)
|
|
63
124
|
@rules.each do |rule|
|
|
64
125
|
next if rule.action == :log
|
|
@@ -74,12 +135,25 @@ module Labkit
|
|
|
74
135
|
rule.match.all? { |key, matcher| matcher.match?(identifier[key]) }
|
|
75
136
|
end
|
|
76
137
|
|
|
77
|
-
def
|
|
138
|
+
def missing_count_distinct_value?(rule, identifier)
|
|
139
|
+
value = identifier[rule.count_distinct]
|
|
140
|
+
value.nil? || value.to_s.empty?
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def evaluate_rule(rule, identifier, cost)
|
|
78
144
|
redis_key = build_redis_key(rule, identifier)
|
|
79
145
|
resolved_limit = Integer(resolve_value(rule.limit))
|
|
80
146
|
resolved_period = Integer(resolve_value(rule.period))
|
|
81
147
|
|
|
82
|
-
|
|
148
|
+
# cost is ignored for count_distinct rules: SADD is binary (a member is
|
|
149
|
+
# either added or not), and the post-add count is SCARD regardless.
|
|
150
|
+
count, ttl =
|
|
151
|
+
if rule.count_distinct
|
|
152
|
+
sadd_with_ttl(redis_key, identifier[rule.count_distinct], resolved_period)
|
|
153
|
+
else
|
|
154
|
+
incr_with_ttl(redis_key, resolved_period, cost)
|
|
155
|
+
end
|
|
156
|
+
|
|
83
157
|
build_result(rule, resolved_limit, resolved_period, count, ttl)
|
|
84
158
|
end
|
|
85
159
|
|
|
@@ -88,7 +162,7 @@ module Labkit
|
|
|
88
162
|
resolved_limit = Integer(resolve_value(rule.limit))
|
|
89
163
|
resolved_period = Integer(resolve_value(rule.period))
|
|
90
164
|
|
|
91
|
-
count, ttl = read_with_ttl(redis_key)
|
|
165
|
+
count, ttl = rule.count_distinct ? scard_with_ttl(redis_key) : read_with_ttl(redis_key)
|
|
92
166
|
build_result(rule, resolved_limit, resolved_period, count, ttl)
|
|
93
167
|
end
|
|
94
168
|
|
|
@@ -133,17 +207,17 @@ module Labkit
|
|
|
133
207
|
end
|
|
134
208
|
end
|
|
135
209
|
|
|
136
|
-
#
|
|
137
|
-
#
|
|
138
|
-
#
|
|
139
|
-
|
|
210
|
+
# Atomically increments the counter by `cost`, sets the TTL on first
|
|
211
|
+
# write, and reads back the post-increment TTL, all in one Redis
|
|
212
|
+
# operation via Lua. See INCR_SCRIPT for the script body.
|
|
213
|
+
#
|
|
214
|
+
# count is parsed as Float because INCRBYFLOAT returns a string-encoded
|
|
215
|
+
# number; the Float is integer-valued when cost is 1, fractional
|
|
216
|
+
# otherwise.
|
|
217
|
+
def incr_with_ttl(redis_key, period, cost)
|
|
140
218
|
@redis.with do |conn|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
pipe.ttl(redis_key)
|
|
144
|
-
end
|
|
145
|
-
conn.expire(redis_key, period) if count == 1
|
|
146
|
-
[count, ttl]
|
|
219
|
+
raw_count, ttl = INCR_SCRIPT.eval(conn, keys: [redis_key], argv: [period, cost])
|
|
220
|
+
[Float(raw_count), ttl]
|
|
147
221
|
end
|
|
148
222
|
end
|
|
149
223
|
|
|
@@ -151,17 +225,45 @@ module Labkit
|
|
|
151
225
|
# A missing key (GET => nil, TTL => -2) is reported as count=0; the
|
|
152
226
|
# build_result fallback then derives reset_at from the rule period
|
|
153
227
|
# since there is no Redis-side window to read.
|
|
228
|
+
#
|
|
229
|
+
# Float parsing accepts both INCR-stored ("5") and INCRBYFLOAT-stored
|
|
230
|
+
# ("5.7") values uniformly.
|
|
154
231
|
def read_with_ttl(redis_key)
|
|
155
232
|
@redis.with do |conn|
|
|
156
233
|
raw_count, ttl = conn.pipelined do |pipe|
|
|
157
234
|
pipe.get(redis_key)
|
|
158
235
|
pipe.ttl(redis_key)
|
|
159
236
|
end
|
|
160
|
-
count = raw_count.nil? ? 0 :
|
|
237
|
+
count = raw_count.nil? ? 0.0 : Float(raw_count)
|
|
161
238
|
[count, ttl]
|
|
162
239
|
end
|
|
163
240
|
end
|
|
164
241
|
|
|
242
|
+
# Atomic SADD + SCARD + conditional EXPIRE in one Redis operation via Lua.
|
|
243
|
+
# See SADD_SCRIPT for the body. Mirrors incr_with_ttl's shape, including
|
|
244
|
+
# the Float-typed count for uniformity with the INCR path.
|
|
245
|
+
def sadd_with_ttl(redis_key, member, period)
|
|
246
|
+
member_str = encode_char_value(member.to_s)
|
|
247
|
+
@redis.with do |conn|
|
|
248
|
+
raw_count, ttl = SADD_SCRIPT.eval(conn, keys: [redis_key], argv: [period, member_str])
|
|
249
|
+
[Float(raw_count), ttl]
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# Pipelined SCARD + TTL. SCARD on a missing key returns 0, so no
|
|
254
|
+
# explicit nil handling is needed (unlike GET in read_with_ttl).
|
|
255
|
+
# SCARD is integer-valued; coerced to Float for type uniformity with
|
|
256
|
+
# the INCR path so callers see a consistent count type.
|
|
257
|
+
def scard_with_ttl(redis_key)
|
|
258
|
+
@redis.with do |conn|
|
|
259
|
+
scard, ttl = conn.pipelined do |pipe|
|
|
260
|
+
pipe.scard(redis_key)
|
|
261
|
+
pipe.ttl(redis_key)
|
|
262
|
+
end
|
|
263
|
+
[Float(scard), ttl]
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
|
|
165
267
|
def log_error(error, identifier)
|
|
166
268
|
@logger.warn(
|
|
167
269
|
message: "rate_limit_error",
|
|
@@ -171,6 +273,16 @@ module Labkit
|
|
|
171
273
|
)
|
|
172
274
|
end
|
|
173
275
|
|
|
276
|
+
def log_missing_count_distinct(rule, identifier)
|
|
277
|
+
@logger.warn(
|
|
278
|
+
message: "rate_limit_missing_count_distinct",
|
|
279
|
+
name: @name,
|
|
280
|
+
rule: rule.name,
|
|
281
|
+
count_distinct: rule.count_distinct.to_s,
|
|
282
|
+
identifier: identifier&.to_h
|
|
283
|
+
)
|
|
284
|
+
end
|
|
285
|
+
|
|
174
286
|
def report_matched_metrics(result)
|
|
175
287
|
Metrics.calls_total.increment(
|
|
176
288
|
rate_limiter: @name,
|
|
@@ -32,10 +32,13 @@ module Labkit
|
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
# @param identifier [Identifier, Hash] caller attributes for this request
|
|
35
|
+
# @param cost [Numeric] amount to add to the counter. Defaults to 1
|
|
36
|
+
# (count-mode). Pass a non-1 Numeric for cost-mode counters such as
|
|
37
|
+
# resource-usage limits; passing 0 reads the counter without writing.
|
|
35
38
|
# @return [Result]
|
|
36
|
-
def check(identifier)
|
|
39
|
+
def check(identifier, cost: 1)
|
|
37
40
|
id = identifier.is_a?(Identifier) ? identifier : Identifier.new(identifier)
|
|
38
|
-
@evaluator.check(id)
|
|
41
|
+
@evaluator.check(id, cost: cost)
|
|
39
42
|
end
|
|
40
43
|
|
|
41
44
|
# Read the current rate-limit state without incrementing the counter.
|
|
@@ -33,13 +33,14 @@ module Labkit
|
|
|
33
33
|
|
|
34
34
|
# Returns RFC-compliant rate limit response headers, or {} when no rule matched or an error occurred.
|
|
35
35
|
# Keys: RateLimit-Limit, RateLimit-Remaining, RateLimit-Reset (Unix timestamp).
|
|
36
|
-
#
|
|
36
|
+
# remaining is coerced to Integer for header output even when info.remaining is fractional;
|
|
37
|
+
# the RateLimit header spec requires integer values.
|
|
37
38
|
def to_response_headers
|
|
38
39
|
return {} unless matched? && !error? && info
|
|
39
40
|
|
|
40
41
|
{
|
|
41
|
-
"RateLimit-Limit" => info.resolved_limit.to_s,
|
|
42
|
-
"RateLimit-Remaining" => info.remaining.to_s,
|
|
42
|
+
"RateLimit-Limit" => info.resolved_limit.to_i.to_s,
|
|
43
|
+
"RateLimit-Remaining" => info.remaining.to_i.to_s,
|
|
43
44
|
"RateLimit-Reset" => info.reset_at.to_i.to_s
|
|
44
45
|
}
|
|
45
46
|
end
|
|
@@ -48,8 +49,12 @@ module Labkit
|
|
|
48
49
|
# Per-window counter data attached to a matched Result.
|
|
49
50
|
# resolved_limit - the evaluated limit Integer for this rule
|
|
50
51
|
# resolved_period - the evaluated period Integer (seconds) for this rule
|
|
51
|
-
# count - the
|
|
52
|
-
#
|
|
52
|
+
# count - the post-increment counter value as a Float; integer-valued
|
|
53
|
+
# for default cost=1 callers, fractional for cost-mode callers.
|
|
54
|
+
# Pre-2.x releases exposed this as Integer; see the migration
|
|
55
|
+
# note in the cost-aware Lua script change.
|
|
56
|
+
# remaining - requests remaining before the limit is hit (floors at 0).
|
|
57
|
+
# Inherits Float typing from count when count is fractional.
|
|
53
58
|
# reset_at - best-effort UTC Time when the counter window resets
|
|
54
59
|
Result::Info = Data.define(:resolved_limit, :resolved_period, :count, :remaining, :reset_at)
|
|
55
60
|
end
|
|
@@ -17,12 +17,32 @@ module Labkit
|
|
|
17
17
|
# (count but always permit; terminates evaluation on match
|
|
18
18
|
# regardless of whether the limit was exceeded)
|
|
19
19
|
# characteristics - identifier keys used to build the compound Redis counter key
|
|
20
|
+
# count_distinct - optional Symbol naming an identifier key. When set, the rule
|
|
21
|
+
# counts the number of distinct values seen for that key within
|
|
22
|
+
# the (characteristics-bucketed) period, backed by a Redis SET.
|
|
23
|
+
# When nil (default), the rule counts the number of calls,
|
|
24
|
+
# backed by INCR. The named key must not overlap +characteristics+.
|
|
20
25
|
#
|
|
21
26
|
# +name+ must be a lowercase alphanumeric-and-underscore string of at most 64
|
|
22
27
|
# characters. It is used as the middle segment of every Redis counter key for
|
|
23
28
|
# this rule, so changing a rule's name mid-window abandons its in-flight counters.
|
|
24
|
-
Rule = Data.define(:name, :match, :limit, :period, :action, :characteristics) do
|
|
25
|
-
def
|
|
29
|
+
Rule = Data.define(:name, :match, :limit, :period, :action, :characteristics, :count_distinct) do
|
|
30
|
+
def self.normalize_count_distinct(value, characteristics_arr)
|
|
31
|
+
sym =
|
|
32
|
+
case value
|
|
33
|
+
when nil then nil
|
|
34
|
+
when Symbol then value
|
|
35
|
+
when String then value.to_sym
|
|
36
|
+
else
|
|
37
|
+
raise ArgumentError, "count_distinct must be a Symbol, String, or nil, got #{value.class}"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
raise ArgumentError, "count_distinct #{sym.inspect} must not overlap characteristics #{characteristics_arr.inspect}" if sym && characteristics_arr.include?(sym)
|
|
41
|
+
|
|
42
|
+
sym
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def initialize(name:, limit:, period:, characteristics:, match: {}, action: :block, count_distinct: nil)
|
|
26
46
|
raise ArgumentError, "name must be a String or Symbol, got #{name.class}" unless name.is_a?(String) || name.is_a?(Symbol)
|
|
27
47
|
|
|
28
48
|
name_str = name.to_s
|
|
@@ -36,13 +56,16 @@ module Labkit
|
|
|
36
56
|
raise ArgumentError, "Rule name too long: #{name.inspect}. Maximum 64 characters" if name_str.length > RULE_NAME_MAX_LENGTH
|
|
37
57
|
end
|
|
38
58
|
|
|
59
|
+
characteristics_arr = Array(characteristics).map(&:to_sym).freeze
|
|
60
|
+
|
|
39
61
|
super(
|
|
40
62
|
name: name_str.freeze,
|
|
41
63
|
match: match.transform_keys(&:to_sym).transform_values { |v| Matcher.build(v) }.freeze,
|
|
42
64
|
limit: limit,
|
|
43
65
|
period: period,
|
|
44
66
|
action: action_sym,
|
|
45
|
-
characteristics:
|
|
67
|
+
characteristics: characteristics_arr,
|
|
68
|
+
count_distinct: self.class.normalize_count_distinct(count_distinct, characteristics_arr)
|
|
46
69
|
)
|
|
47
70
|
end
|
|
48
71
|
end
|
data/lib/labkit/rate_limit.rb
CHANGED
|
@@ -43,9 +43,10 @@ module Labkit
|
|
|
43
43
|
# @param rules [Array<Rule>] ordered list of rules (first match wins)
|
|
44
44
|
# @param redis [Object, nil] Redis client; falls back to config.redis
|
|
45
45
|
# @param logger [Logger, nil] logger; falls back to config.logger
|
|
46
|
+
# @param cost [Numeric] amount to add to the counter; see Limiter#check
|
|
46
47
|
# @return [Result]
|
|
47
|
-
def check(name:, identifier:, rules:, redis: nil, logger: nil)
|
|
48
|
-
Limiter.new(name: name, rules: rules, redis: redis, logger: logger).check(identifier)
|
|
48
|
+
def check(name:, identifier:, rules:, redis: nil, logger: nil, cost: 1)
|
|
49
|
+
Limiter.new(name: name, rules: rules, redis: redis, logger: logger).check(identifier, cost: cost)
|
|
49
50
|
end
|
|
50
51
|
end
|
|
51
52
|
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "openssl"
|
|
4
|
+
require "redis"
|
|
5
|
+
|
|
6
|
+
module Labkit
|
|
7
|
+
module Redis
|
|
8
|
+
# Wraps a Lua script for EVALSHA-with-NOSCRIPT-fallback execution.
|
|
9
|
+
# The SHA is computed once at construction. Redis caches the script
|
|
10
|
+
# body the first time EVAL is invoked; subsequent EVALSHA calls hit
|
|
11
|
+
# that cache. SCRIPT FLUSH or a Redis restart drops the cache; the
|
|
12
|
+
# NOSCRIPT recovery re-ships the body and re-populates it.
|
|
13
|
+
#
|
|
14
|
+
# @example
|
|
15
|
+
# SCRIPT = Labkit::Redis::Script.new(<<~LUA)
|
|
16
|
+
# return redis.call('INCRBY', KEYS[1], ARGV[1])
|
|
17
|
+
# LUA
|
|
18
|
+
#
|
|
19
|
+
# pool.with { |conn| SCRIPT.eval(conn, keys: ["counter"], argv: [1]) }
|
|
20
|
+
class Script
|
|
21
|
+
attr_reader :body, :sha
|
|
22
|
+
|
|
23
|
+
def initialize(body)
|
|
24
|
+
@body = body.freeze
|
|
25
|
+
# SHA1 is mandated by the Redis EVALSHA wire protocol, not a discretionary hash choice.
|
|
26
|
+
@sha = OpenSSL::Digest::SHA1.hexdigest(body).freeze # rubocop:disable Fips/SHA1
|
|
27
|
+
freeze
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @param conn a Redis client (the connection checked out of a pool)
|
|
31
|
+
# @param keys [Array] KEYS arguments to the Lua script
|
|
32
|
+
# @param argv [Array] ARGV arguments to the Lua script
|
|
33
|
+
# @return the script's return value
|
|
34
|
+
def eval(conn, keys:, argv:)
|
|
35
|
+
conn.evalsha(@sha, keys: keys, argv: argv)
|
|
36
|
+
rescue ::Redis::CommandError => e
|
|
37
|
+
raise unless e.message.start_with?("NOSCRIPT")
|
|
38
|
+
|
|
39
|
+
conn.eval(@body, keys: keys, argv: argv)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
data/lib/labkit/redis.rb
ADDED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gitlab-labkit
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 2.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Andrew Newdigate
|
|
@@ -18,7 +18,7 @@ dependencies:
|
|
|
18
18
|
version: 5.0.0
|
|
19
19
|
- - "<"
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
|
-
version: 8.
|
|
21
|
+
version: 8.2.0
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -28,7 +28,7 @@ dependencies:
|
|
|
28
28
|
version: 5.0.0
|
|
29
29
|
- - "<"
|
|
30
30
|
- !ruby/object:Gem::Version
|
|
31
|
-
version: 8.
|
|
31
|
+
version: 8.2.0
|
|
32
32
|
- !ruby/object:Gem::Dependency
|
|
33
33
|
name: activesupport
|
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -38,7 +38,7 @@ dependencies:
|
|
|
38
38
|
version: 5.0.0
|
|
39
39
|
- - "<"
|
|
40
40
|
- !ruby/object:Gem::Version
|
|
41
|
-
version: 8.
|
|
41
|
+
version: 8.2.0
|
|
42
42
|
type: :runtime
|
|
43
43
|
prerelease: false
|
|
44
44
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -48,7 +48,7 @@ dependencies:
|
|
|
48
48
|
version: 5.0.0
|
|
49
49
|
- - "<"
|
|
50
50
|
- !ruby/object:Gem::Version
|
|
51
|
-
version: 8.
|
|
51
|
+
version: 8.2.0
|
|
52
52
|
- !ruby/object:Gem::Dependency
|
|
53
53
|
name: grpc
|
|
54
54
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -135,58 +135,82 @@ dependencies:
|
|
|
135
135
|
name: opentelemetry-sdk
|
|
136
136
|
requirement: !ruby/object:Gem::Requirement
|
|
137
137
|
requirements:
|
|
138
|
-
- - "
|
|
138
|
+
- - ">="
|
|
139
139
|
- !ruby/object:Gem::Version
|
|
140
140
|
version: '1.10'
|
|
141
|
+
- - "<"
|
|
142
|
+
- !ruby/object:Gem::Version
|
|
143
|
+
version: '2'
|
|
141
144
|
type: :runtime
|
|
142
145
|
prerelease: false
|
|
143
146
|
version_requirements: !ruby/object:Gem::Requirement
|
|
144
147
|
requirements:
|
|
145
|
-
- - "
|
|
148
|
+
- - ">="
|
|
146
149
|
- !ruby/object:Gem::Version
|
|
147
150
|
version: '1.10'
|
|
151
|
+
- - "<"
|
|
152
|
+
- !ruby/object:Gem::Version
|
|
153
|
+
version: '2'
|
|
148
154
|
- !ruby/object:Gem::Dependency
|
|
149
155
|
name: opentelemetry-instrumentation-all
|
|
150
156
|
requirement: !ruby/object:Gem::Requirement
|
|
151
157
|
requirements:
|
|
152
|
-
- - "
|
|
158
|
+
- - ">="
|
|
159
|
+
- !ruby/object:Gem::Version
|
|
160
|
+
version: '0.89'
|
|
161
|
+
- - "<"
|
|
153
162
|
- !ruby/object:Gem::Version
|
|
154
|
-
version:
|
|
163
|
+
version: '1'
|
|
155
164
|
type: :runtime
|
|
156
165
|
prerelease: false
|
|
157
166
|
version_requirements: !ruby/object:Gem::Requirement
|
|
158
167
|
requirements:
|
|
159
|
-
- - "
|
|
168
|
+
- - ">="
|
|
169
|
+
- !ruby/object:Gem::Version
|
|
170
|
+
version: '0.89'
|
|
171
|
+
- - "<"
|
|
160
172
|
- !ruby/object:Gem::Version
|
|
161
|
-
version:
|
|
173
|
+
version: '1'
|
|
162
174
|
- !ruby/object:Gem::Dependency
|
|
163
175
|
name: opentelemetry-exporter-otlp
|
|
164
176
|
requirement: !ruby/object:Gem::Requirement
|
|
165
177
|
requirements:
|
|
166
|
-
- - "
|
|
178
|
+
- - ">="
|
|
179
|
+
- !ruby/object:Gem::Version
|
|
180
|
+
version: '0.31'
|
|
181
|
+
- - "<"
|
|
167
182
|
- !ruby/object:Gem::Version
|
|
168
|
-
version:
|
|
183
|
+
version: '1'
|
|
169
184
|
type: :runtime
|
|
170
185
|
prerelease: false
|
|
171
186
|
version_requirements: !ruby/object:Gem::Requirement
|
|
172
187
|
requirements:
|
|
173
|
-
- - "
|
|
188
|
+
- - ">="
|
|
189
|
+
- !ruby/object:Gem::Version
|
|
190
|
+
version: '0.31'
|
|
191
|
+
- - "<"
|
|
174
192
|
- !ruby/object:Gem::Version
|
|
175
|
-
version:
|
|
193
|
+
version: '1'
|
|
176
194
|
- !ruby/object:Gem::Dependency
|
|
177
195
|
name: opentracing
|
|
178
196
|
requirement: !ruby/object:Gem::Requirement
|
|
179
197
|
requirements:
|
|
180
|
-
- - "
|
|
198
|
+
- - ">="
|
|
181
199
|
- !ruby/object:Gem::Version
|
|
182
200
|
version: '0.4'
|
|
201
|
+
- - "<"
|
|
202
|
+
- !ruby/object:Gem::Version
|
|
203
|
+
version: '1'
|
|
183
204
|
type: :runtime
|
|
184
205
|
prerelease: false
|
|
185
206
|
version_requirements: !ruby/object:Gem::Requirement
|
|
186
207
|
requirements:
|
|
187
|
-
- - "
|
|
208
|
+
- - ">="
|
|
188
209
|
- !ruby/object:Gem::Version
|
|
189
210
|
version: '0.4'
|
|
211
|
+
- - "<"
|
|
212
|
+
- !ruby/object:Gem::Version
|
|
213
|
+
version: '1'
|
|
190
214
|
- !ruby/object:Gem::Dependency
|
|
191
215
|
name: pg_query
|
|
192
216
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -410,7 +434,7 @@ dependencies:
|
|
|
410
434
|
version: 5.0.0
|
|
411
435
|
- - "<"
|
|
412
436
|
- !ruby/object:Gem::Version
|
|
413
|
-
version: 8.
|
|
437
|
+
version: 8.2.0
|
|
414
438
|
type: :development
|
|
415
439
|
prerelease: false
|
|
416
440
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -420,7 +444,7 @@ dependencies:
|
|
|
420
444
|
version: 5.0.0
|
|
421
445
|
- - "<"
|
|
422
446
|
- !ruby/object:Gem::Version
|
|
423
|
-
version: 8.
|
|
447
|
+
version: 8.2.0
|
|
424
448
|
- !ruby/object:Gem::Dependency
|
|
425
449
|
name: rake
|
|
426
450
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -523,6 +547,7 @@ files:
|
|
|
523
547
|
- ".env.example.sh"
|
|
524
548
|
- ".gitignore"
|
|
525
549
|
- ".gitlab-ci-asdf-versions.yml"
|
|
550
|
+
- ".gitlab-ci-other-versions.yml"
|
|
526
551
|
- ".gitlab-ci.yml"
|
|
527
552
|
- ".gitlab/CODEOWNERS"
|
|
528
553
|
- ".gitlab/merge_request_templates/default.md"
|
|
@@ -609,6 +634,8 @@ files:
|
|
|
609
634
|
- lib/labkit/rate_limit/metrics.rb
|
|
610
635
|
- lib/labkit/rate_limit/result.rb
|
|
611
636
|
- lib/labkit/rate_limit/rule.rb
|
|
637
|
+
- lib/labkit/redis.rb
|
|
638
|
+
- lib/labkit/redis/script.rb
|
|
612
639
|
- lib/labkit/rspec/README.md
|
|
613
640
|
- lib/labkit/rspec/matchers.rb
|
|
614
641
|
- lib/labkit/rspec/matchers/user_experience_matchers.rb
|