throttle_machines 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/throttle_machines/controller_helpers.rb +2 -2
- data/lib/throttle_machines/engine.rb +1 -6
- data/lib/throttle_machines/hedged_breaker.rb +2 -2
- data/lib/throttle_machines/hedged_request.rb +2 -2
- data/lib/throttle_machines/instrumentation.rb +19 -19
- data/lib/throttle_machines/middleware.rb +1 -2
- data/lib/throttle_machines/rack_middleware/allow2_ban.rb +10 -4
- data/lib/throttle_machines/rack_middleware/configuration.rb +14 -14
- data/lib/throttle_machines/rack_middleware/fail2_ban.rb +19 -11
- data/lib/throttle_machines/rack_middleware.rb +2 -2
- data/lib/throttle_machines/storage/base.rb +2 -23
- data/lib/throttle_machines/storage/memory.rb +7 -113
- data/lib/throttle_machines/storage/null.rb +0 -21
- data/lib/throttle_machines/storage/redis.rb +5 -68
- data/lib/throttle_machines/version.rb +1 -1
- data/lib/throttle_machines.rb +32 -19
- metadata +5 -8
- data/lib/throttle_machines/storage/redis/get_breaker_state.lua +0 -23
- data/lib/throttle_machines/storage/redis/record_breaker_failure.lua +0 -24
- data/lib/throttle_machines/storage/redis/record_breaker_success.lua +0 -16
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ff1906a9e597bee94979f637bf351aa53160ec2b349005ed731069bdb85cd032
|
|
4
|
+
data.tar.gz: 2293e5d59855593686cc2a8df0df560c5286452bc63bbcea4e13fca04d824c92
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7c1fa93375186f6036c1efcba1292207ba0c831ee0d5e5bb73b7c985e2d45b3c52362d79d590ad1904a25568040e00e5929373381fb48da57155a646e8588e24
|
|
7
|
+
data.tar.gz: ae5ffa6881f0e7913227de2991ae8ffd17af31525040e43741491691cddde6ee8e4d27a9a2ad5dd5d2537335237e008f3397e9c4d3f45fab123cdac850ee7c73
|
|
@@ -21,10 +21,10 @@ module ThrottleMachines
|
|
|
21
21
|
set_rate_limit_headers(limiter)
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
def with_throttle(key = nil, limit:, period:, &)
|
|
24
|
+
def with_throttle(key = nil, limit:, period:, &block)
|
|
25
25
|
key ||= default_throttle_key
|
|
26
26
|
|
|
27
|
-
ThrottleMachines.limit(key, limit: limit, period: period, &)
|
|
27
|
+
ThrottleMachines.limit(key, limit: limit, period: period, &block)
|
|
28
28
|
rescue ThrottledError => e
|
|
29
29
|
render_rate_limited(e.limiter)
|
|
30
30
|
end
|
|
@@ -13,11 +13,6 @@ module ThrottleMachines
|
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
ThrottleMachines.configure do |config|
|
|
18
|
-
# Use Rails.cache , user can override
|
|
19
|
-
config.store = Rails.cache
|
|
20
|
-
end
|
|
21
|
-
end
|
|
16
|
+
# No default Rails.cache binding; storage is managed by ThrottleMachines
|
|
22
17
|
end
|
|
23
18
|
end
|
|
@@ -15,7 +15,7 @@ module ThrottleMachines
|
|
|
15
15
|
@backend ||= ActiveSupport::Notifications
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
def instrument(event_name, payload = {}, &)
|
|
18
|
+
def instrument(event_name, payload = {}, &block)
|
|
19
19
|
if !enabled || backend.nil?
|
|
20
20
|
return yield if block_given?
|
|
21
21
|
|
|
@@ -23,7 +23,7 @@ module ThrottleMachines
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
full_event_name = "#{event_name}.throttle_machines"
|
|
26
|
-
backend.instrument(full_event_name, payload, &)
|
|
26
|
+
backend.instrument(full_event_name, payload, &block)
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
# Convenience methods for common events
|
|
@@ -66,9 +66,9 @@ module ThrottleMachines
|
|
|
66
66
|
# Circuit breaker events
|
|
67
67
|
def circuit_opened(breaker, failure_count:)
|
|
68
68
|
payload = {
|
|
69
|
-
key: breaker.
|
|
70
|
-
failure_threshold: breaker.failure_threshold,
|
|
71
|
-
timeout: breaker.
|
|
69
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
70
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
71
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil),
|
|
72
72
|
failure_count: failure_count
|
|
73
73
|
}
|
|
74
74
|
instrument('circuit_breaker.opened', payload)
|
|
@@ -76,35 +76,35 @@ module ThrottleMachines
|
|
|
76
76
|
|
|
77
77
|
def circuit_closed(breaker)
|
|
78
78
|
payload = {
|
|
79
|
-
key: breaker.
|
|
80
|
-
failure_threshold: breaker.failure_threshold,
|
|
81
|
-
timeout: breaker.
|
|
79
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
80
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
81
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil)
|
|
82
82
|
}
|
|
83
83
|
instrument('circuit_breaker.closed', payload)
|
|
84
84
|
end
|
|
85
85
|
|
|
86
86
|
def circuit_half_opened(breaker)
|
|
87
87
|
payload = {
|
|
88
|
-
key: breaker.
|
|
89
|
-
failure_threshold: breaker.failure_threshold,
|
|
90
|
-
timeout: breaker.
|
|
91
|
-
half_open_requests: breaker.
|
|
88
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
89
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
90
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil),
|
|
91
|
+
half_open_requests: (breaker.respond_to?(:configuration) ? breaker.configuration[:half_open_calls] : nil)
|
|
92
92
|
}
|
|
93
93
|
instrument('circuit_breaker.half_opened', payload)
|
|
94
94
|
end
|
|
95
95
|
|
|
96
96
|
def circuit_success(breaker)
|
|
97
97
|
payload = {
|
|
98
|
-
key: breaker.
|
|
99
|
-
state: breaker.state
|
|
98
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
99
|
+
state: (breaker.respond_to?(:to_h) ? breaker.to_h[:state] : nil)
|
|
100
100
|
}
|
|
101
101
|
instrument('circuit_breaker.success', payload)
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
def circuit_failure(breaker, error: nil)
|
|
105
105
|
payload = {
|
|
106
|
-
key: breaker.
|
|
107
|
-
state: breaker.state,
|
|
106
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
107
|
+
state: (breaker.respond_to?(:to_h) ? breaker.to_h[:state] : nil),
|
|
108
108
|
error_class: error&.class&.name,
|
|
109
109
|
error_message: error&.message
|
|
110
110
|
}
|
|
@@ -113,9 +113,9 @@ module ThrottleMachines
|
|
|
113
113
|
|
|
114
114
|
def circuit_rejected(breaker)
|
|
115
115
|
payload = {
|
|
116
|
-
key: breaker.
|
|
117
|
-
failure_threshold: breaker.failure_threshold,
|
|
118
|
-
timeout: breaker.
|
|
116
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
117
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
118
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil)
|
|
119
119
|
}
|
|
120
120
|
instrument('circuit_breaker.rejected', payload)
|
|
121
121
|
end
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module ThrottleMachines
|
|
4
4
|
class Middleware
|
|
5
|
-
def initialize(app,
|
|
5
|
+
def initialize(app, &config_block)
|
|
6
6
|
@app = app
|
|
7
|
-
@store = store || ThrottleMachines.configuration.store
|
|
8
7
|
@rules = []
|
|
9
8
|
|
|
10
9
|
instance_eval(&config_block) if config_block
|
|
@@ -34,12 +34,18 @@ module ThrottleMachines
|
|
|
34
34
|
|
|
35
35
|
# Check if we've had enough successful requests
|
|
36
36
|
if success_limiter.remaining.zero?
|
|
37
|
-
# Reset the fail2ban breaker
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
# Reset the fail2ban breaker (use BreakerMachines circuit)
|
|
38
|
+
breaker = BreakerMachines::Registry.instance.get_or_create_dynamic_circuit(
|
|
39
|
+
fail_key,
|
|
40
|
+
self,
|
|
41
|
+
failure_threshold: @maxretry,
|
|
42
|
+
failure_window: @findtime,
|
|
43
|
+
reset_timeout: @bantime
|
|
44
|
+
)
|
|
45
|
+
breaker.hard_reset
|
|
40
46
|
|
|
41
47
|
# Reset our own counter
|
|
42
|
-
storage.reset_counter(success_key, @findtime)
|
|
48
|
+
ThrottleMachines.storage.reset_counter(success_key, @findtime)
|
|
43
49
|
else
|
|
44
50
|
# Increment success counter
|
|
45
51
|
begin
|
|
@@ -32,27 +32,27 @@ module ThrottleMachines
|
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
# DSL Methods
|
|
35
|
-
def throttle(name, options = {}, &)
|
|
36
|
-
@throttles[name] = Throttle.new(name, options, &)
|
|
35
|
+
def throttle(name, options = {}, &block)
|
|
36
|
+
@throttles[name] = Throttle.new(name, options, &block)
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
def track(name, options = {}, &)
|
|
40
|
-
@tracks[name] = Track.new(name, options, &)
|
|
39
|
+
def track(name, options = {}, &block)
|
|
40
|
+
@tracks[name] = Track.new(name, options, &block)
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
def safelist(name = nil, &)
|
|
43
|
+
def safelist(name = nil, &block)
|
|
44
44
|
if name
|
|
45
|
-
@safelists[name] = Safelist.new(name, &)
|
|
45
|
+
@safelists[name] = Safelist.new(name, &block)
|
|
46
46
|
else
|
|
47
|
-
@anonymous_safelists << Safelist.new(nil, &)
|
|
47
|
+
@anonymous_safelists << Safelist.new(nil, &block)
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
def blocklist(name = nil, &)
|
|
51
|
+
def blocklist(name = nil, &block)
|
|
52
52
|
if name
|
|
53
|
-
@blocklists[name] = Blocklist.new(name, &)
|
|
53
|
+
@blocklists[name] = Blocklist.new(name, &block)
|
|
54
54
|
else
|
|
55
|
-
@anonymous_blocklists << Blocklist.new(nil, &)
|
|
55
|
+
@anonymous_blocklists << Blocklist.new(nil, &block)
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
|
|
@@ -64,12 +64,12 @@ module ThrottleMachines
|
|
|
64
64
|
@anonymous_blocklists << Blocklist.new(nil) { |req| req.ip == ip_address }
|
|
65
65
|
end
|
|
66
66
|
|
|
67
|
-
def fail2ban(name, options = {}, &)
|
|
68
|
-
@fail2bans[name] = Fail2Ban.new(name, options, &)
|
|
67
|
+
def fail2ban(name, options = {}, &block)
|
|
68
|
+
@fail2bans[name] = Fail2Ban.new(name, options, &block)
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
-
def allow2ban(name, options = {}, &)
|
|
72
|
-
@allow2bans[name] = Allow2Ban.new(name, options, &)
|
|
71
|
+
def allow2ban(name, options = {}, &block)
|
|
72
|
+
@allow2bans[name] = Allow2Ban.new(name, options, &block)
|
|
73
73
|
end
|
|
74
74
|
|
|
75
75
|
# Check methods
|
|
@@ -20,18 +20,25 @@ module ThrottleMachines
|
|
|
20
20
|
|
|
21
21
|
key = "fail2ban:#{@name}:#{discriminator}"
|
|
22
22
|
|
|
23
|
-
# Use circuit
|
|
24
|
-
breaker =
|
|
23
|
+
# Use a globally managed BreakerMachines circuit as the ban mechanism
|
|
24
|
+
breaker = BreakerMachines::Registry.instance.get_or_create_dynamic_circuit(
|
|
25
25
|
key,
|
|
26
|
+
self,
|
|
26
27
|
failure_threshold: @maxretry,
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
failure_window: @findtime,
|
|
29
|
+
reset_timeout: @bantime
|
|
29
30
|
)
|
|
30
31
|
|
|
31
32
|
# Check if circuit is open (banned)
|
|
32
33
|
if breaker.open?
|
|
33
|
-
|
|
34
|
-
|
|
34
|
+
stats = breaker.stats
|
|
35
|
+
now = BreakerMachines.monotonic_time
|
|
36
|
+
time_until_unban = if stats.opened_at
|
|
37
|
+
remaining = @bantime - (now - stats.opened_at)
|
|
38
|
+
remaining.positive? ? remaining : 0
|
|
39
|
+
else
|
|
40
|
+
@bantime
|
|
41
|
+
end
|
|
35
42
|
|
|
36
43
|
request.env['rack.attack.matched'] = @name
|
|
37
44
|
request.env['rack.attack.match_type'] = :fail2ban
|
|
@@ -41,8 +48,8 @@ module ThrottleMachines
|
|
|
41
48
|
maxretry: @maxretry,
|
|
42
49
|
findtime: @findtime,
|
|
43
50
|
bantime: @bantime,
|
|
44
|
-
failures:
|
|
45
|
-
time_until_unban:
|
|
51
|
+
failures: stats.failure_count,
|
|
52
|
+
time_until_unban: time_until_unban
|
|
46
53
|
}
|
|
47
54
|
|
|
48
55
|
ThrottleMachines::RackMiddleware.instrument(request)
|
|
@@ -61,11 +68,12 @@ module ThrottleMachines
|
|
|
61
68
|
# Use the breaker to record a failure if block returns true
|
|
62
69
|
return unless yield
|
|
63
70
|
|
|
64
|
-
breaker =
|
|
71
|
+
breaker = BreakerMachines::Registry.instance.get_or_create_dynamic_circuit(
|
|
65
72
|
key,
|
|
73
|
+
self,
|
|
66
74
|
failure_threshold: @maxretry,
|
|
67
|
-
|
|
68
|
-
|
|
75
|
+
failure_window: @findtime,
|
|
76
|
+
reset_timeout: @bantime
|
|
69
77
|
)
|
|
70
78
|
|
|
71
79
|
# Record failure by trying to call through the breaker
|
|
@@ -30,9 +30,9 @@ module ThrottleMachines
|
|
|
30
30
|
:safelists,
|
|
31
31
|
:blocklists
|
|
32
32
|
|
|
33
|
-
def configure(&)
|
|
33
|
+
def configure(&block)
|
|
34
34
|
@configuration ||= Configuration.new
|
|
35
|
-
@configuration.instance_eval(&) if block
|
|
35
|
+
@configuration.instance_eval(&block) if block
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
# rubocop:disable Rails/Delegate -- Ruby 3.4 compatibility issue with delegate
|
|
@@ -42,27 +42,6 @@ module ThrottleMachines
|
|
|
42
42
|
raise NotImplementedError
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
# Circuit breaker operations
|
|
46
|
-
def get_breaker_state(key)
|
|
47
|
-
raise NotImplementedError
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def record_breaker_success(key, timeout, half_open_requests = 1)
|
|
51
|
-
raise NotImplementedError
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def record_breaker_failure(key, threshold, timeout)
|
|
55
|
-
raise NotImplementedError
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def trip_breaker(key, timeout)
|
|
59
|
-
raise NotImplementedError
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def reset_breaker(key)
|
|
63
|
-
raise NotImplementedError
|
|
64
|
-
end
|
|
65
|
-
|
|
66
45
|
# Utility operations
|
|
67
46
|
def clear(pattern = nil)
|
|
68
47
|
raise NotImplementedError
|
|
@@ -72,8 +51,8 @@ module ThrottleMachines
|
|
|
72
51
|
raise NotImplementedError
|
|
73
52
|
end
|
|
74
53
|
|
|
75
|
-
def with_timeout(timeout, &)
|
|
76
|
-
Timeout.timeout(timeout, &)
|
|
54
|
+
def with_timeout(timeout, &block)
|
|
55
|
+
Timeout.timeout(timeout, &block)
|
|
77
56
|
rescue Timeout::Error
|
|
78
57
|
nil
|
|
79
58
|
end
|
|
@@ -10,7 +10,6 @@ module ThrottleMachines
|
|
|
10
10
|
@counters = Concurrent::Hash.new
|
|
11
11
|
@gcra_states = Concurrent::Hash.new
|
|
12
12
|
@token_buckets = Concurrent::Hash.new
|
|
13
|
-
@breaker_states = Concurrent::Hash.new
|
|
14
13
|
|
|
15
14
|
# Use a striped lock pattern - pool of locks for fine-grained concurrency
|
|
16
15
|
@lock_pool_size = options[:lock_pool_size] || 32
|
|
@@ -180,98 +179,7 @@ module ThrottleMachines
|
|
|
180
179
|
end
|
|
181
180
|
end
|
|
182
181
|
|
|
183
|
-
#
|
|
184
|
-
def get_breaker_state(key)
|
|
185
|
-
# First try with read lock
|
|
186
|
-
state = with_read_lock("breaker:#{key}") do
|
|
187
|
-
@breaker_states[key] || { state: :closed, failures: 0, last_failure: nil }
|
|
188
|
-
end
|
|
189
|
-
|
|
190
|
-
# Check if we need to transition from open to half-open
|
|
191
|
-
if state[:state] == :open && state[:opens_at] && current_time >= state[:opens_at]
|
|
192
|
-
# Release read lock and acquire write lock
|
|
193
|
-
with_write_lock("breaker:#{key}") do
|
|
194
|
-
# Re-check condition after acquiring write lock
|
|
195
|
-
current_state = @breaker_states[key]
|
|
196
|
-
if current_state && current_state[:state] == :open && current_state[:opens_at] && current_time >= current_state[:opens_at]
|
|
197
|
-
@breaker_states[key] = current_state.merge(
|
|
198
|
-
state: :half_open,
|
|
199
|
-
half_open_attempts: 0
|
|
200
|
-
)
|
|
201
|
-
end
|
|
202
|
-
@breaker_states[key] || { state: :closed, failures: 0, last_failure: nil }
|
|
203
|
-
end
|
|
204
|
-
else
|
|
205
|
-
state
|
|
206
|
-
end
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
def record_breaker_success(key, _timeout, half_open_requests = 1)
|
|
210
|
-
with_write_lock("breaker:#{key}") do
|
|
211
|
-
state = @breaker_states[key]
|
|
212
|
-
return unless state
|
|
213
|
-
|
|
214
|
-
case state[:state]
|
|
215
|
-
when :half_open
|
|
216
|
-
attempts = (state[:half_open_attempts] || 0) + 1
|
|
217
|
-
if attempts >= half_open_requests
|
|
218
|
-
@breaker_states.delete(key)
|
|
219
|
-
else
|
|
220
|
-
@breaker_states[key] = state.merge(half_open_attempts: attempts)
|
|
221
|
-
end
|
|
222
|
-
when :closed
|
|
223
|
-
# Reset failure count on success
|
|
224
|
-
@breaker_states[key] = state.merge(failures: 0) if state[:failures].positive?
|
|
225
|
-
end
|
|
226
|
-
end
|
|
227
|
-
end
|
|
228
|
-
|
|
229
|
-
def record_breaker_failure(key, threshold, timeout)
|
|
230
|
-
with_write_lock("breaker:#{key}") do
|
|
231
|
-
state = @breaker_states[key] || { state: :closed, failures: 0 }
|
|
232
|
-
now = current_time
|
|
233
|
-
|
|
234
|
-
case state[:state]
|
|
235
|
-
when :closed
|
|
236
|
-
failures = state[:failures] + 1
|
|
237
|
-
@breaker_states[key] = if failures >= threshold
|
|
238
|
-
{
|
|
239
|
-
state: :open,
|
|
240
|
-
failures: failures,
|
|
241
|
-
last_failure: now,
|
|
242
|
-
opens_at: now + timeout
|
|
243
|
-
}
|
|
244
|
-
else
|
|
245
|
-
state.merge(failures: failures, last_failure: now)
|
|
246
|
-
end
|
|
247
|
-
when :half_open
|
|
248
|
-
@breaker_states[key] = {
|
|
249
|
-
state: :open,
|
|
250
|
-
failures: state[:failures],
|
|
251
|
-
last_failure: now,
|
|
252
|
-
opens_at: now + timeout
|
|
253
|
-
}
|
|
254
|
-
end
|
|
255
|
-
|
|
256
|
-
@breaker_states[key]
|
|
257
|
-
end
|
|
258
|
-
end
|
|
259
|
-
|
|
260
|
-
def trip_breaker(key, timeout)
|
|
261
|
-
with_write_lock("breaker:#{key}") do
|
|
262
|
-
now = current_time
|
|
263
|
-
@breaker_states[key] = {
|
|
264
|
-
state: :open,
|
|
265
|
-
failures: 0,
|
|
266
|
-
last_failure: now,
|
|
267
|
-
opens_at: now + timeout
|
|
268
|
-
}
|
|
269
|
-
end
|
|
270
|
-
end
|
|
271
|
-
|
|
272
|
-
def reset_breaker(key)
|
|
273
|
-
with_write_lock("breaker:#{key}") { @breaker_states.delete(key) }
|
|
274
|
-
end
|
|
182
|
+
# No circuit breaker operations here: breaker state is owned by BreakerMachines
|
|
275
183
|
|
|
276
184
|
# Utility operations
|
|
277
185
|
def clear(pattern = nil)
|
|
@@ -279,7 +187,7 @@ module ThrottleMachines
|
|
|
279
187
|
regex = Regexp.new(pattern.gsub('*', '.*'))
|
|
280
188
|
|
|
281
189
|
# Clear matching keys from all stores
|
|
282
|
-
[@counters, @gcra_states, @token_buckets
|
|
190
|
+
[@counters, @gcra_states, @token_buckets].each do |store|
|
|
283
191
|
store.each_key do |k|
|
|
284
192
|
store.delete(k) if k&.match?(regex)
|
|
285
193
|
end
|
|
@@ -288,7 +196,6 @@ module ThrottleMachines
|
|
|
288
196
|
@counters.clear
|
|
289
197
|
@gcra_states.clear
|
|
290
198
|
@token_buckets.clear
|
|
291
|
-
@breaker_states.clear
|
|
292
199
|
end
|
|
293
200
|
end
|
|
294
201
|
|
|
@@ -305,12 +212,12 @@ module ThrottleMachines
|
|
|
305
212
|
|
|
306
213
|
private
|
|
307
214
|
|
|
308
|
-
def with_read_lock(key, &)
|
|
309
|
-
lock_for(key).with_read_lock(&)
|
|
215
|
+
def with_read_lock(key, &block)
|
|
216
|
+
lock_for(key).with_read_lock(&block)
|
|
310
217
|
end
|
|
311
218
|
|
|
312
|
-
def with_write_lock(key, &)
|
|
313
|
-
lock_for(key).with_write_lock(&)
|
|
219
|
+
def with_write_lock(key, &block)
|
|
220
|
+
lock_for(key).with_write_lock(&block)
|
|
314
221
|
end
|
|
315
222
|
|
|
316
223
|
def lock_for(key)
|
|
@@ -350,20 +257,7 @@ module ThrottleMachines
|
|
|
350
257
|
with_write_lock(key) { @token_buckets.delete(key) } if data[:expires_at] && data[:expires_at] <= now
|
|
351
258
|
end
|
|
352
259
|
|
|
353
|
-
#
|
|
354
|
-
@breaker_states.each_pair do |key, data|
|
|
355
|
-
should_delete = false
|
|
356
|
-
|
|
357
|
-
# Clean closed states that have been idle
|
|
358
|
-
should_delete = true if data[:state] == :closed && data[:failures].zero?
|
|
359
|
-
|
|
360
|
-
# Clean expired open states (older than 2x timeout)
|
|
361
|
-
if data[:opens_at] && now > data[:opens_at] + ((data[:opens_at] - (data[:last_failure] || now)) * 2)
|
|
362
|
-
should_delete = true
|
|
363
|
-
end
|
|
364
|
-
|
|
365
|
-
with_write_lock("breaker:#{key}") { @breaker_states.delete(key) } if should_delete
|
|
366
|
-
end
|
|
260
|
+
# No breaker state cleanup: breaker state is managed by BreakerMachines
|
|
367
261
|
rescue StandardError => e
|
|
368
262
|
# Log error but don't crash cleanup thread
|
|
369
263
|
warn "ThrottleMachines: Cleanup error: #{e.message}"
|
|
@@ -54,27 +54,6 @@ module ThrottleMachines
|
|
|
54
54
|
}
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
# Circuit breaker operations
|
|
58
|
-
def get_breaker_state(_key)
|
|
59
|
-
{ state: :closed, failures: 0, last_failure: nil }
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def record_breaker_success(_key, _timeout, _half_open_requests = 1)
|
|
63
|
-
true
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def record_breaker_failure(_key, _threshold, _timeout)
|
|
67
|
-
{ state: :closed, failures: 0, last_failure: nil }
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
def trip_breaker(_key, _timeout)
|
|
71
|
-
true
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def reset_breaker(_key)
|
|
75
|
-
true
|
|
76
|
-
end
|
|
77
|
-
|
|
78
57
|
# Utility operations
|
|
79
58
|
def clear(_pattern = nil)
|
|
80
59
|
true
|
|
@@ -11,9 +11,7 @@ module ThrottleMachines
|
|
|
11
11
|
PEEK_GCRA_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'peek_gcra.lua'))
|
|
12
12
|
PEEK_TOKEN_BUCKET_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'peek_token_bucket.lua'))
|
|
13
13
|
INCREMENT_COUNTER_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'increment_counter.lua'))
|
|
14
|
-
|
|
15
|
-
RECORD_BREAKER_SUCCESS_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'record_breaker_success.lua'))
|
|
16
|
-
RECORD_BREAKER_FAILURE_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'record_breaker_failure.lua'))
|
|
14
|
+
# Breaker scripts removed: breaker state is owned by BreakerMachines
|
|
17
15
|
|
|
18
16
|
def initialize(options = {})
|
|
19
17
|
super
|
|
@@ -165,68 +163,7 @@ module ThrottleMachines
|
|
|
165
163
|
retry
|
|
166
164
|
end
|
|
167
165
|
|
|
168
|
-
#
|
|
169
|
-
def get_breaker_state(key)
|
|
170
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
171
|
-
|
|
172
|
-
# Use Lua script for atomic read and potential state transition
|
|
173
|
-
result = with_redis do |redis|
|
|
174
|
-
redis.eval(GET_BREAKER_STATE_SCRIPT, keys: [breaker_key], argv: [current_time])
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
return { state: :closed, failures: 0, last_failure: nil } if result.empty?
|
|
178
|
-
|
|
179
|
-
# Convert hash from Lua to Ruby format
|
|
180
|
-
state = {}
|
|
181
|
-
result.each_slice(2) { |k, v| state[k] = v }
|
|
182
|
-
|
|
183
|
-
{
|
|
184
|
-
state: state['state'].to_sym,
|
|
185
|
-
failures: state['failures'].to_i,
|
|
186
|
-
last_failure: state['last_failure']&.to_f,
|
|
187
|
-
opens_at: state['opens_at']&.to_f,
|
|
188
|
-
half_open_attempts: state['half_open_attempts']&.to_i
|
|
189
|
-
}
|
|
190
|
-
end
|
|
191
|
-
|
|
192
|
-
def record_breaker_success(key, _timeout, half_open_requests = 1)
|
|
193
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
194
|
-
|
|
195
|
-
# Use Lua script for atomic success recording
|
|
196
|
-
with_redis do |redis|
|
|
197
|
-
redis.eval(RECORD_BREAKER_SUCCESS_SCRIPT, keys: [breaker_key], argv: [half_open_requests])
|
|
198
|
-
end
|
|
199
|
-
end
|
|
200
|
-
|
|
201
|
-
def record_breaker_failure(key, threshold, timeout)
|
|
202
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
203
|
-
now = current_time
|
|
204
|
-
|
|
205
|
-
# Use Lua script for atomic failure recording
|
|
206
|
-
with_redis do |redis|
|
|
207
|
-
redis.eval(RECORD_BREAKER_FAILURE_SCRIPT, keys: [breaker_key], argv: [threshold, timeout, now])
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
get_breaker_state(key)
|
|
211
|
-
end
|
|
212
|
-
|
|
213
|
-
def trip_breaker(key, timeout)
|
|
214
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
215
|
-
now = current_time
|
|
216
|
-
|
|
217
|
-
with_redis do |redis|
|
|
218
|
-
redis.hmset(breaker_key,
|
|
219
|
-
'state', 'open',
|
|
220
|
-
'failures', 0,
|
|
221
|
-
'last_failure', now,
|
|
222
|
-
'opens_at', now + timeout)
|
|
223
|
-
redis.expire(breaker_key, (timeout * 2).to_i)
|
|
224
|
-
end
|
|
225
|
-
end
|
|
226
|
-
|
|
227
|
-
def reset_breaker(key)
|
|
228
|
-
with_redis { |r| r.del(prefixed("breaker:#{key}")) }
|
|
229
|
-
end
|
|
166
|
+
# No circuit breaker operations: breaker state is owned by BreakerMachines
|
|
230
167
|
|
|
231
168
|
# Utility operations
|
|
232
169
|
def clear(pattern = nil)
|
|
@@ -280,13 +217,13 @@ module ThrottleMachines
|
|
|
280
217
|
raise ArgumentError, "Invalid Redis connection: #{e.message}"
|
|
281
218
|
end
|
|
282
219
|
|
|
283
|
-
def with_redis(&)
|
|
220
|
+
def with_redis(&block)
|
|
284
221
|
if @redis.respond_to?(:with)
|
|
285
222
|
# Connection pool
|
|
286
|
-
@redis.with(&)
|
|
223
|
+
@redis.with(&block)
|
|
287
224
|
else
|
|
288
225
|
# Regular Redis client
|
|
289
|
-
|
|
226
|
+
block.call(@redis)
|
|
290
227
|
end
|
|
291
228
|
end
|
|
292
229
|
end
|
data/lib/throttle_machines.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'json'
|
|
4
4
|
require 'timeout'
|
|
5
5
|
require 'zeitwerk'
|
|
6
|
-
require 'active_support/
|
|
6
|
+
require 'active_support/core_ext/class/attribute'
|
|
7
7
|
|
|
8
8
|
# Ecosystem dependencies
|
|
9
9
|
require 'chrono_machines'
|
|
@@ -15,21 +15,31 @@ loader.ignore("#{__dir__}/throttle_machines/engine.rb") unless defined?(Rails::E
|
|
|
15
15
|
loader.setup
|
|
16
16
|
|
|
17
17
|
module ThrottleMachines
|
|
18
|
-
|
|
18
|
+
class Configuration
|
|
19
|
+
attr_accessor :default_limit, :default_period, :default_storage, :clock,
|
|
20
|
+
:instrumentation_enabled, :instrumentation_backend, :_storage_instance
|
|
21
|
+
|
|
22
|
+
def initialize
|
|
23
|
+
@default_limit = 100
|
|
24
|
+
@default_period = 60 # 1 minute
|
|
25
|
+
@default_storage = :memory
|
|
26
|
+
@clock = nil
|
|
27
|
+
@instrumentation_enabled = true
|
|
28
|
+
@instrumentation_backend = nil
|
|
29
|
+
@_storage_instance = nil
|
|
30
|
+
end
|
|
31
|
+
end
|
|
19
32
|
|
|
20
|
-
|
|
21
|
-
config_accessor :default_limit, default: 100
|
|
22
|
-
config_accessor :default_period, default: 60 # 1 minute
|
|
23
|
-
config_accessor :default_storage, default: :memory
|
|
24
|
-
config_accessor :clock, default: nil
|
|
25
|
-
config_accessor :instrumentation_enabled, default: true
|
|
26
|
-
config_accessor :instrumentation_backend
|
|
27
|
-
config_accessor :_storage_instance
|
|
33
|
+
@config = Configuration.new
|
|
28
34
|
|
|
29
35
|
class << self
|
|
30
36
|
# Delegate monotonic time to BreakerMachines for consistency
|
|
31
37
|
delegate :monotonic_time, to: :BreakerMachines
|
|
32
38
|
|
|
39
|
+
def config
|
|
40
|
+
@config
|
|
41
|
+
end
|
|
42
|
+
|
|
33
43
|
def configure
|
|
34
44
|
yield(config) if block_given?
|
|
35
45
|
|
|
@@ -65,28 +75,28 @@ module ThrottleMachines
|
|
|
65
75
|
control
|
|
66
76
|
end
|
|
67
77
|
|
|
68
|
-
def limit(key, limit:, period:, algorithm: :fixed_window, &)
|
|
78
|
+
def limit(key, limit:, period:, algorithm: :fixed_window, &block)
|
|
69
79
|
limiter = limiter(key, limit: limit, period: period, algorithm: algorithm)
|
|
70
|
-
limiter.throttle!(&)
|
|
80
|
+
limiter.throttle!(&block)
|
|
71
81
|
end
|
|
72
82
|
|
|
73
|
-
def break_circuit(key, failures:, timeout:, &)
|
|
74
|
-
# Delegate to
|
|
83
|
+
def break_circuit(key, failures:, timeout:, &block)
|
|
84
|
+
# Delegate to BreakerMachines; use reset_timeout for open duration
|
|
75
85
|
breaker = BreakerMachines::Circuit.new(
|
|
76
|
-
key
|
|
86
|
+
key,
|
|
77
87
|
failure_threshold: failures,
|
|
78
|
-
|
|
88
|
+
reset_timeout: timeout
|
|
79
89
|
)
|
|
80
|
-
breaker.call(&)
|
|
90
|
+
breaker.call(&block)
|
|
81
91
|
end
|
|
82
92
|
|
|
83
|
-
def retry_with(max_attempts: 3, backoff: :exponential, &)
|
|
93
|
+
def retry_with(max_attempts: 3, backoff: :exponential, &block)
|
|
84
94
|
# Delegate to chrono_machines
|
|
85
95
|
policy_options = {
|
|
86
96
|
max_attempts: max_attempts,
|
|
87
97
|
jitter_factor: backoff == :exponential ? 1.0 : 0.0
|
|
88
98
|
}
|
|
89
|
-
ChronoMachines.retry(policy_options, &)
|
|
99
|
+
ChronoMachines.retry(policy_options, &block)
|
|
90
100
|
end
|
|
91
101
|
|
|
92
102
|
def limiter(key, limit:, period:, algorithm: :fixed_window)
|
|
@@ -131,4 +141,7 @@ module ThrottleMachines
|
|
|
131
141
|
|
|
132
142
|
CircuitOpenError = BreakerMachines::CircuitOpenError
|
|
133
143
|
RetryExhaustedError = ChronoMachines::MaxRetriesExceededError
|
|
144
|
+
|
|
145
|
+
# Back-compat wrapper: use BreakerMachines as the circuit implementation
|
|
146
|
+
Breaker = BreakerMachines::Circuit
|
|
134
147
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: throttle_machines
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Abdelkader Boudih
|
|
@@ -15,14 +15,14 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version:
|
|
18
|
+
version: 8.0.4
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - ">="
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version:
|
|
25
|
+
version: 8.0.4
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
27
|
name: concurrent-ruby
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -71,14 +71,14 @@ dependencies:
|
|
|
71
71
|
requirements:
|
|
72
72
|
- - "~>"
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
|
-
version: '0.
|
|
74
|
+
version: '0.7'
|
|
75
75
|
type: :runtime
|
|
76
76
|
prerelease: false
|
|
77
77
|
version_requirements: !ruby/object:Gem::Requirement
|
|
78
78
|
requirements:
|
|
79
79
|
- - "~>"
|
|
80
80
|
- !ruby/object:Gem::Version
|
|
81
|
-
version: '0.
|
|
81
|
+
version: '0.7'
|
|
82
82
|
- !ruby/object:Gem::Dependency
|
|
83
83
|
name: chrono_machines
|
|
84
84
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -129,12 +129,9 @@ files:
|
|
|
129
129
|
- lib/throttle_machines/storage/null.rb
|
|
130
130
|
- lib/throttle_machines/storage/redis.rb
|
|
131
131
|
- lib/throttle_machines/storage/redis/gcra.lua
|
|
132
|
-
- lib/throttle_machines/storage/redis/get_breaker_state.lua
|
|
133
132
|
- lib/throttle_machines/storage/redis/increment_counter.lua
|
|
134
133
|
- lib/throttle_machines/storage/redis/peek_gcra.lua
|
|
135
134
|
- lib/throttle_machines/storage/redis/peek_token_bucket.lua
|
|
136
|
-
- lib/throttle_machines/storage/redis/record_breaker_failure.lua
|
|
137
|
-
- lib/throttle_machines/storage/redis/record_breaker_success.lua
|
|
138
135
|
- lib/throttle_machines/storage/redis/token_bucket.lua
|
|
139
136
|
- lib/throttle_machines/throttled_error.rb
|
|
140
137
|
- lib/throttle_machines/version.rb
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
local data = redis.call('HGETALL', KEYS[1])
|
|
2
|
-
if #data == 0 then
|
|
3
|
-
return {}
|
|
4
|
-
end
|
|
5
|
-
|
|
6
|
-
local state = {}
|
|
7
|
-
for i = 1, #data, 2 do
|
|
8
|
-
state[data[i]] = data[i + 1]
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
-- Auto-transition from open to half-open if timeout passed
|
|
12
|
-
if state['state'] == 'open' and state['opens_at'] then
|
|
13
|
-
local now = tonumber(ARGV[1])
|
|
14
|
-
local opens_at = tonumber(state['opens_at'])
|
|
15
|
-
|
|
16
|
-
if now >= opens_at then
|
|
17
|
-
redis.call('HSET', KEYS[1], 'state', 'half_open', 'half_open_attempts', '0')
|
|
18
|
-
state['state'] = 'half_open'
|
|
19
|
-
state['half_open_attempts'] = '0'
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
return state
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
local state = redis.call('HGET', KEYS[1], 'state') or 'closed'
|
|
2
|
-
local now = ARGV[3]
|
|
3
|
-
local timeout = tonumber(ARGV[2])
|
|
4
|
-
|
|
5
|
-
if state == 'half_open' then
|
|
6
|
-
-- Failure in half-open state, just re-open the circuit
|
|
7
|
-
redis.call('HMSET', KEYS[1],
|
|
8
|
-
'state', 'open',
|
|
9
|
-
'opens_at', tonumber(now) + timeout,
|
|
10
|
-
'last_failure', now
|
|
11
|
-
)
|
|
12
|
-
else -- state is 'closed' or nil
|
|
13
|
-
local failures = redis.call('HINCRBY', KEYS[1], 'failures', 1)
|
|
14
|
-
redis.call('HSET', KEYS[1], 'last_failure', now)
|
|
15
|
-
|
|
16
|
-
if failures >= tonumber(ARGV[1]) then
|
|
17
|
-
redis.call('HMSET', KEYS[1],
|
|
18
|
-
'state', 'open',
|
|
19
|
-
'opens_at', tonumber(now) + timeout
|
|
20
|
-
)
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
redis.call('EXPIRE', KEYS[1], timeout * 2)
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
local state = redis.call('HGET', KEYS[1], 'state')
|
|
2
|
-
|
|
3
|
-
if state == 'half_open' then
|
|
4
|
-
-- Increment half-open attempts and potentially close the circuit
|
|
5
|
-
local attempts = redis.call('HINCRBY', KEYS[1], 'half_open_attempts', 1)
|
|
6
|
-
|
|
7
|
-
if attempts >= tonumber(ARGV[1]) then
|
|
8
|
-
redis.call('DEL', KEYS[1])
|
|
9
|
-
end
|
|
10
|
-
elseif state == 'closed' then
|
|
11
|
-
-- Reset failure count on success in closed state
|
|
12
|
-
local failures = redis.call('HGET', KEYS[1], 'failures')
|
|
13
|
-
if failures and tonumber(failures) > 0 then
|
|
14
|
-
redis.call('HSET', KEYS[1], 'failures', 0)
|
|
15
|
-
end
|
|
16
|
-
end
|