throttle_machines 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/throttle_machines/controller_helpers.rb +3 -3
- data/lib/throttle_machines/engine.rb +1 -8
- data/lib/throttle_machines/hedged_breaker.rb +23 -0
- data/lib/throttle_machines/hedged_request.rb +2 -22
- data/lib/throttle_machines/instrumentation.rb +20 -24
- data/lib/throttle_machines/middleware.rb +1 -2
- data/lib/throttle_machines/rack_middleware/allow2_ban.rb +10 -4
- data/lib/throttle_machines/rack_middleware/configuration.rb +14 -14
- data/lib/throttle_machines/rack_middleware/fail2_ban.rb +19 -11
- data/lib/throttle_machines/rack_middleware.rb +5 -3
- data/lib/throttle_machines/storage/base.rb +2 -23
- data/lib/throttle_machines/storage/memory.rb +7 -114
- data/lib/throttle_machines/storage/null.rb +0 -23
- data/lib/throttle_machines/storage/redis/gcra.lua +22 -0
- data/lib/throttle_machines/storage/redis/increment_counter.lua +9 -0
- data/lib/throttle_machines/storage/redis/peek_gcra.lua +16 -0
- data/lib/throttle_machines/storage/redis/peek_token_bucket.lua +18 -0
- data/lib/throttle_machines/storage/redis/token_bucket.lua +23 -0
- data/lib/throttle_machines/storage/redis.rb +13 -233
- data/lib/throttle_machines/version.rb +1 -1
- data/lib/throttle_machines.rb +34 -25
- metadata +12 -7
- data/lib/throttle_machines/clock.rb +0 -41
- /data/{MIT-LICENSE → LICENSE} +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ff1906a9e597bee94979f637bf351aa53160ec2b349005ed731069bdb85cd032
|
|
4
|
+
data.tar.gz: 2293e5d59855593686cc2a8df0df560c5286452bc63bbcea4e13fca04d824c92
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7c1fa93375186f6036c1efcba1292207ba0c831ee0d5e5bb73b7c985e2d45b3c52362d79d590ad1904a25568040e00e5929373381fb48da57155a646e8588e24
|
|
7
|
+
data.tar.gz: ae5ffa6881f0e7913227de2991ae8ffd17af31525040e43741491691cddde6ee8e4d27a9a2ad5dd5d2537335237e008f3397e9c4d3f45fab123cdac850ee7c73
|
|
@@ -5,7 +5,7 @@ module ThrottleMachines
|
|
|
5
5
|
extend ActiveSupport::Concern
|
|
6
6
|
|
|
7
7
|
included do
|
|
8
|
-
if respond_to?(:helper_method)
|
|
8
|
+
if respond_to?(:helper_method) # No available in API Mode
|
|
9
9
|
helper_method :rate_limited?
|
|
10
10
|
helper_method :rate_limit_remaining
|
|
11
11
|
end
|
|
@@ -21,10 +21,10 @@ module ThrottleMachines
|
|
|
21
21
|
set_rate_limit_headers(limiter)
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
-
def with_throttle(key = nil, limit:, period:, &)
|
|
24
|
+
def with_throttle(key = nil, limit:, period:, &block)
|
|
25
25
|
key ||= default_throttle_key
|
|
26
26
|
|
|
27
|
-
ThrottleMachines.limit(key, limit: limit, period: period, &)
|
|
27
|
+
ThrottleMachines.limit(key, limit: limit, period: period, &block)
|
|
28
28
|
rescue ThrottledError => e
|
|
29
29
|
render_rate_limited(e.limiter)
|
|
30
30
|
end
|
|
@@ -13,13 +13,6 @@ module ThrottleMachines
|
|
|
13
13
|
end
|
|
14
14
|
end
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
ThrottleMachines.configure do |config|
|
|
18
|
-
# Use Redis if available in Rails cache
|
|
19
|
-
if defined?(Redis) && Rails.cache.respond_to?(:redis)
|
|
20
|
-
config.store = ThrottleMachines::Stores::Redis.new(Rails.cache.redis)
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
end
|
|
16
|
+
# No default Rails.cache binding; storage is managed by ThrottleMachines
|
|
24
17
|
end
|
|
25
18
|
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ThrottleMachines
|
|
4
|
+
# Hedged request with circuit breaker integration
|
|
5
|
+
class HedgedBreaker
|
|
6
|
+
def initialize(breakers, delay: 0.05)
|
|
7
|
+
@breakers = Array(breakers)
|
|
8
|
+
@hedged = HedgedRequest.new(
|
|
9
|
+
delay: delay,
|
|
10
|
+
max_attempts: @breakers.size
|
|
11
|
+
)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def run(&block)
|
|
15
|
+
@hedged.run do |attempt|
|
|
16
|
+
breaker = @breakers[attempt]
|
|
17
|
+
next if breaker.nil?
|
|
18
|
+
|
|
19
|
+
breaker.call(&block)
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -94,8 +94,8 @@ module ThrottleMachines
|
|
|
94
94
|
end
|
|
95
95
|
|
|
96
96
|
# Run async version
|
|
97
|
-
def run_async(&)
|
|
98
|
-
Concurrent::Promises.future { run(&) }
|
|
97
|
+
def run_async(&block)
|
|
98
|
+
Concurrent::Promises.future { run(&block) }
|
|
99
99
|
end
|
|
100
100
|
|
|
101
101
|
# Shutdown the executor
|
|
@@ -105,26 +105,6 @@ module ThrottleMachines
|
|
|
105
105
|
end
|
|
106
106
|
end
|
|
107
107
|
|
|
108
|
-
# Hedged request with circuit breaker integration
|
|
109
|
-
class HedgedBreaker
|
|
110
|
-
def initialize(breakers, delay: 0.05)
|
|
111
|
-
@breakers = Array(breakers)
|
|
112
|
-
@hedged = HedgedRequest.new(
|
|
113
|
-
delay: delay,
|
|
114
|
-
max_attempts: @breakers.size
|
|
115
|
-
)
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
def run(&)
|
|
119
|
-
@hedged.run do |attempt|
|
|
120
|
-
breaker = @breakers[attempt]
|
|
121
|
-
next if breaker.nil?
|
|
122
|
-
|
|
123
|
-
breaker.call(&)
|
|
124
|
-
end
|
|
125
|
-
end
|
|
126
|
-
end
|
|
127
|
-
|
|
128
108
|
# Convenience method
|
|
129
109
|
def self.hedged_request(**, &)
|
|
130
110
|
hedged = HedgedRequest.new(**)
|
|
@@ -12,14 +12,10 @@ module ThrottleMachines
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
def backend
|
|
15
|
-
@backend ||=
|
|
16
|
-
ActiveSupport::Notifications
|
|
17
|
-
else
|
|
18
|
-
NullBackend.new
|
|
19
|
-
end
|
|
15
|
+
@backend ||= ActiveSupport::Notifications
|
|
20
16
|
end
|
|
21
17
|
|
|
22
|
-
def instrument(event_name, payload = {}, &)
|
|
18
|
+
def instrument(event_name, payload = {}, &block)
|
|
23
19
|
if !enabled || backend.nil?
|
|
24
20
|
return yield if block_given?
|
|
25
21
|
|
|
@@ -27,7 +23,7 @@ module ThrottleMachines
|
|
|
27
23
|
end
|
|
28
24
|
|
|
29
25
|
full_event_name = "#{event_name}.throttle_machines"
|
|
30
|
-
backend.instrument(full_event_name, payload, &)
|
|
26
|
+
backend.instrument(full_event_name, payload, &block)
|
|
31
27
|
end
|
|
32
28
|
|
|
33
29
|
# Convenience methods for common events
|
|
@@ -70,9 +66,9 @@ module ThrottleMachines
|
|
|
70
66
|
# Circuit breaker events
|
|
71
67
|
def circuit_opened(breaker, failure_count:)
|
|
72
68
|
payload = {
|
|
73
|
-
key: breaker.
|
|
74
|
-
failure_threshold: breaker.failure_threshold,
|
|
75
|
-
timeout: breaker.
|
|
69
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
70
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
71
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil),
|
|
76
72
|
failure_count: failure_count
|
|
77
73
|
}
|
|
78
74
|
instrument('circuit_breaker.opened', payload)
|
|
@@ -80,35 +76,35 @@ module ThrottleMachines
|
|
|
80
76
|
|
|
81
77
|
def circuit_closed(breaker)
|
|
82
78
|
payload = {
|
|
83
|
-
key: breaker.
|
|
84
|
-
failure_threshold: breaker.failure_threshold,
|
|
85
|
-
timeout: breaker.
|
|
79
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
80
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
81
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil)
|
|
86
82
|
}
|
|
87
83
|
instrument('circuit_breaker.closed', payload)
|
|
88
84
|
end
|
|
89
85
|
|
|
90
86
|
def circuit_half_opened(breaker)
|
|
91
87
|
payload = {
|
|
92
|
-
key: breaker.
|
|
93
|
-
failure_threshold: breaker.failure_threshold,
|
|
94
|
-
timeout: breaker.
|
|
95
|
-
half_open_requests: breaker.
|
|
88
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
89
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
90
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil),
|
|
91
|
+
half_open_requests: (breaker.respond_to?(:configuration) ? breaker.configuration[:half_open_calls] : nil)
|
|
96
92
|
}
|
|
97
93
|
instrument('circuit_breaker.half_opened', payload)
|
|
98
94
|
end
|
|
99
95
|
|
|
100
96
|
def circuit_success(breaker)
|
|
101
97
|
payload = {
|
|
102
|
-
key: breaker.
|
|
103
|
-
state: breaker.state
|
|
98
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
99
|
+
state: (breaker.respond_to?(:to_h) ? breaker.to_h[:state] : nil)
|
|
104
100
|
}
|
|
105
101
|
instrument('circuit_breaker.success', payload)
|
|
106
102
|
end
|
|
107
103
|
|
|
108
104
|
def circuit_failure(breaker, error: nil)
|
|
109
105
|
payload = {
|
|
110
|
-
key: breaker.
|
|
111
|
-
state: breaker.state,
|
|
106
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
107
|
+
state: (breaker.respond_to?(:to_h) ? breaker.to_h[:state] : nil),
|
|
112
108
|
error_class: error&.class&.name,
|
|
113
109
|
error_message: error&.message
|
|
114
110
|
}
|
|
@@ -117,9 +113,9 @@ module ThrottleMachines
|
|
|
117
113
|
|
|
118
114
|
def circuit_rejected(breaker)
|
|
119
115
|
payload = {
|
|
120
|
-
key: breaker.
|
|
121
|
-
failure_threshold: breaker.failure_threshold,
|
|
122
|
-
timeout: breaker.
|
|
116
|
+
key: (breaker.respond_to?(:name) ? breaker.name : breaker.to_s),
|
|
117
|
+
failure_threshold: (breaker.respond_to?(:configuration) ? breaker.configuration[:failure_threshold] : nil),
|
|
118
|
+
timeout: (breaker.respond_to?(:configuration) ? breaker.configuration[:reset_timeout] : nil)
|
|
123
119
|
}
|
|
124
120
|
instrument('circuit_breaker.rejected', payload)
|
|
125
121
|
end
|
|
@@ -2,9 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module ThrottleMachines
|
|
4
4
|
class Middleware
|
|
5
|
-
def initialize(app,
|
|
5
|
+
def initialize(app, &config_block)
|
|
6
6
|
@app = app
|
|
7
|
-
@store = store || ThrottleMachines.configuration.store
|
|
8
7
|
@rules = []
|
|
9
8
|
|
|
10
9
|
instance_eval(&config_block) if config_block
|
|
@@ -34,12 +34,18 @@ module ThrottleMachines
|
|
|
34
34
|
|
|
35
35
|
# Check if we've had enough successful requests
|
|
36
36
|
if success_limiter.remaining.zero?
|
|
37
|
-
# Reset the fail2ban breaker
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
# Reset the fail2ban breaker (use BreakerMachines circuit)
|
|
38
|
+
breaker = BreakerMachines::Registry.instance.get_or_create_dynamic_circuit(
|
|
39
|
+
fail_key,
|
|
40
|
+
self,
|
|
41
|
+
failure_threshold: @maxretry,
|
|
42
|
+
failure_window: @findtime,
|
|
43
|
+
reset_timeout: @bantime
|
|
44
|
+
)
|
|
45
|
+
breaker.hard_reset
|
|
40
46
|
|
|
41
47
|
# Reset our own counter
|
|
42
|
-
storage.reset_counter(success_key, @findtime)
|
|
48
|
+
ThrottleMachines.storage.reset_counter(success_key, @findtime)
|
|
43
49
|
else
|
|
44
50
|
# Increment success counter
|
|
45
51
|
begin
|
|
@@ -32,27 +32,27 @@ module ThrottleMachines
|
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
# DSL Methods
|
|
35
|
-
def throttle(name, options = {}, &)
|
|
36
|
-
@throttles[name] = Throttle.new(name, options, &)
|
|
35
|
+
def throttle(name, options = {}, &block)
|
|
36
|
+
@throttles[name] = Throttle.new(name, options, &block)
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
def track(name, options = {}, &)
|
|
40
|
-
@tracks[name] = Track.new(name, options, &)
|
|
39
|
+
def track(name, options = {}, &block)
|
|
40
|
+
@tracks[name] = Track.new(name, options, &block)
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
def safelist(name = nil, &)
|
|
43
|
+
def safelist(name = nil, &block)
|
|
44
44
|
if name
|
|
45
|
-
@safelists[name] = Safelist.new(name, &)
|
|
45
|
+
@safelists[name] = Safelist.new(name, &block)
|
|
46
46
|
else
|
|
47
|
-
@anonymous_safelists << Safelist.new(nil, &)
|
|
47
|
+
@anonymous_safelists << Safelist.new(nil, &block)
|
|
48
48
|
end
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
def blocklist(name = nil, &)
|
|
51
|
+
def blocklist(name = nil, &block)
|
|
52
52
|
if name
|
|
53
|
-
@blocklists[name] = Blocklist.new(name, &)
|
|
53
|
+
@blocklists[name] = Blocklist.new(name, &block)
|
|
54
54
|
else
|
|
55
|
-
@anonymous_blocklists << Blocklist.new(nil, &)
|
|
55
|
+
@anonymous_blocklists << Blocklist.new(nil, &block)
|
|
56
56
|
end
|
|
57
57
|
end
|
|
58
58
|
|
|
@@ -64,12 +64,12 @@ module ThrottleMachines
|
|
|
64
64
|
@anonymous_blocklists << Blocklist.new(nil) { |req| req.ip == ip_address }
|
|
65
65
|
end
|
|
66
66
|
|
|
67
|
-
def fail2ban(name, options = {}, &)
|
|
68
|
-
@fail2bans[name] = Fail2Ban.new(name, options, &)
|
|
67
|
+
def fail2ban(name, options = {}, &block)
|
|
68
|
+
@fail2bans[name] = Fail2Ban.new(name, options, &block)
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
-
def allow2ban(name, options = {}, &)
|
|
72
|
-
@allow2bans[name] = Allow2Ban.new(name, options, &)
|
|
71
|
+
def allow2ban(name, options = {}, &block)
|
|
72
|
+
@allow2bans[name] = Allow2Ban.new(name, options, &block)
|
|
73
73
|
end
|
|
74
74
|
|
|
75
75
|
# Check methods
|
|
@@ -20,18 +20,25 @@ module ThrottleMachines
|
|
|
20
20
|
|
|
21
21
|
key = "fail2ban:#{@name}:#{discriminator}"
|
|
22
22
|
|
|
23
|
-
# Use circuit
|
|
24
|
-
breaker =
|
|
23
|
+
# Use a globally managed BreakerMachines circuit as the ban mechanism
|
|
24
|
+
breaker = BreakerMachines::Registry.instance.get_or_create_dynamic_circuit(
|
|
25
25
|
key,
|
|
26
|
+
self,
|
|
26
27
|
failure_threshold: @maxretry,
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
failure_window: @findtime,
|
|
29
|
+
reset_timeout: @bantime
|
|
29
30
|
)
|
|
30
31
|
|
|
31
32
|
# Check if circuit is open (banned)
|
|
32
33
|
if breaker.open?
|
|
33
|
-
|
|
34
|
-
|
|
34
|
+
stats = breaker.stats
|
|
35
|
+
now = BreakerMachines.monotonic_time
|
|
36
|
+
time_until_unban = if stats.opened_at
|
|
37
|
+
remaining = @bantime - (now - stats.opened_at)
|
|
38
|
+
remaining.positive? ? remaining : 0
|
|
39
|
+
else
|
|
40
|
+
@bantime
|
|
41
|
+
end
|
|
35
42
|
|
|
36
43
|
request.env['rack.attack.matched'] = @name
|
|
37
44
|
request.env['rack.attack.match_type'] = :fail2ban
|
|
@@ -41,8 +48,8 @@ module ThrottleMachines
|
|
|
41
48
|
maxretry: @maxretry,
|
|
42
49
|
findtime: @findtime,
|
|
43
50
|
bantime: @bantime,
|
|
44
|
-
failures:
|
|
45
|
-
time_until_unban:
|
|
51
|
+
failures: stats.failure_count,
|
|
52
|
+
time_until_unban: time_until_unban
|
|
46
53
|
}
|
|
47
54
|
|
|
48
55
|
ThrottleMachines::RackMiddleware.instrument(request)
|
|
@@ -61,11 +68,12 @@ module ThrottleMachines
|
|
|
61
68
|
# Use the breaker to record a failure if block returns true
|
|
62
69
|
return unless yield
|
|
63
70
|
|
|
64
|
-
breaker =
|
|
71
|
+
breaker = BreakerMachines::Registry.instance.get_or_create_dynamic_circuit(
|
|
65
72
|
key,
|
|
73
|
+
self,
|
|
66
74
|
failure_threshold: @maxretry,
|
|
67
|
-
|
|
68
|
-
|
|
75
|
+
failure_window: @findtime,
|
|
76
|
+
reset_timeout: @bantime
|
|
69
77
|
)
|
|
70
78
|
|
|
71
79
|
# Record failure by trying to call through the breaker
|
|
@@ -30,14 +30,16 @@ module ThrottleMachines
|
|
|
30
30
|
:safelists,
|
|
31
31
|
:blocklists
|
|
32
32
|
|
|
33
|
-
def configure(&)
|
|
33
|
+
def configure(&block)
|
|
34
34
|
@configuration ||= Configuration.new
|
|
35
|
-
@configuration.instance_eval(&) if block
|
|
35
|
+
@configuration.instance_eval(&block) if block
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
+
# rubocop:disable Rails/Delegate -- Ruby 3.4 compatibility issue with delegate
|
|
38
39
|
def reset!
|
|
39
40
|
ThrottleMachines.reset!
|
|
40
41
|
end
|
|
42
|
+
# rubocop:enable Rails/Delegate
|
|
41
43
|
|
|
42
44
|
def clear!
|
|
43
45
|
@configuration = Configuration.new
|
|
@@ -54,7 +56,7 @@ module ThrottleMachines
|
|
|
54
56
|
|
|
55
57
|
# Set defaults
|
|
56
58
|
@enabled = true
|
|
57
|
-
@notifier = ActiveSupport::Notifications
|
|
59
|
+
@notifier = ActiveSupport::Notifications
|
|
58
60
|
@configuration = Configuration.new
|
|
59
61
|
|
|
60
62
|
def initialize(app)
|
|
@@ -42,27 +42,6 @@ module ThrottleMachines
|
|
|
42
42
|
raise NotImplementedError
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
# Circuit breaker operations
|
|
46
|
-
def get_breaker_state(key)
|
|
47
|
-
raise NotImplementedError
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def record_breaker_success(key, timeout, half_open_requests = 1)
|
|
51
|
-
raise NotImplementedError
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
def record_breaker_failure(key, threshold, timeout)
|
|
55
|
-
raise NotImplementedError
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def trip_breaker(key, timeout)
|
|
59
|
-
raise NotImplementedError
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def reset_breaker(key)
|
|
63
|
-
raise NotImplementedError
|
|
64
|
-
end
|
|
65
|
-
|
|
66
45
|
# Utility operations
|
|
67
46
|
def clear(pattern = nil)
|
|
68
47
|
raise NotImplementedError
|
|
@@ -72,8 +51,8 @@ module ThrottleMachines
|
|
|
72
51
|
raise NotImplementedError
|
|
73
52
|
end
|
|
74
53
|
|
|
75
|
-
def with_timeout(timeout, &)
|
|
76
|
-
Timeout.timeout(timeout, &)
|
|
54
|
+
def with_timeout(timeout, &block)
|
|
55
|
+
Timeout.timeout(timeout, &block)
|
|
77
56
|
rescue Timeout::Error
|
|
78
57
|
nil
|
|
79
58
|
end
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative 'base'
|
|
4
3
|
require 'concurrent'
|
|
5
4
|
|
|
6
5
|
module ThrottleMachines
|
|
@@ -11,7 +10,6 @@ module ThrottleMachines
|
|
|
11
10
|
@counters = Concurrent::Hash.new
|
|
12
11
|
@gcra_states = Concurrent::Hash.new
|
|
13
12
|
@token_buckets = Concurrent::Hash.new
|
|
14
|
-
@breaker_states = Concurrent::Hash.new
|
|
15
13
|
|
|
16
14
|
# Use a striped lock pattern - pool of locks for fine-grained concurrency
|
|
17
15
|
@lock_pool_size = options[:lock_pool_size] || 32
|
|
@@ -181,98 +179,7 @@ module ThrottleMachines
|
|
|
181
179
|
end
|
|
182
180
|
end
|
|
183
181
|
|
|
184
|
-
#
|
|
185
|
-
def get_breaker_state(key)
|
|
186
|
-
# First try with read lock
|
|
187
|
-
state = with_read_lock("breaker:#{key}") do
|
|
188
|
-
@breaker_states[key] || { state: :closed, failures: 0, last_failure: nil }
|
|
189
|
-
end
|
|
190
|
-
|
|
191
|
-
# Check if we need to transition from open to half-open
|
|
192
|
-
if state[:state] == :open && state[:opens_at] && current_time >= state[:opens_at]
|
|
193
|
-
# Release read lock and acquire write lock
|
|
194
|
-
with_write_lock("breaker:#{key}") do
|
|
195
|
-
# Re-check condition after acquiring write lock
|
|
196
|
-
current_state = @breaker_states[key]
|
|
197
|
-
if current_state && current_state[:state] == :open && current_state[:opens_at] && current_time >= current_state[:opens_at]
|
|
198
|
-
@breaker_states[key] = current_state.merge(
|
|
199
|
-
state: :half_open,
|
|
200
|
-
half_open_attempts: 0
|
|
201
|
-
)
|
|
202
|
-
end
|
|
203
|
-
@breaker_states[key] || { state: :closed, failures: 0, last_failure: nil }
|
|
204
|
-
end
|
|
205
|
-
else
|
|
206
|
-
state
|
|
207
|
-
end
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
def record_breaker_success(key, _timeout, half_open_requests = 1)
|
|
211
|
-
with_write_lock("breaker:#{key}") do
|
|
212
|
-
state = @breaker_states[key]
|
|
213
|
-
return unless state
|
|
214
|
-
|
|
215
|
-
case state[:state]
|
|
216
|
-
when :half_open
|
|
217
|
-
attempts = (state[:half_open_attempts] || 0) + 1
|
|
218
|
-
if attempts >= half_open_requests
|
|
219
|
-
@breaker_states.delete(key)
|
|
220
|
-
else
|
|
221
|
-
@breaker_states[key] = state.merge(half_open_attempts: attempts)
|
|
222
|
-
end
|
|
223
|
-
when :closed
|
|
224
|
-
# Reset failure count on success
|
|
225
|
-
@breaker_states[key] = state.merge(failures: 0) if state[:failures].positive?
|
|
226
|
-
end
|
|
227
|
-
end
|
|
228
|
-
end
|
|
229
|
-
|
|
230
|
-
def record_breaker_failure(key, threshold, timeout)
|
|
231
|
-
with_write_lock("breaker:#{key}") do
|
|
232
|
-
state = @breaker_states[key] || { state: :closed, failures: 0 }
|
|
233
|
-
now = current_time
|
|
234
|
-
|
|
235
|
-
case state[:state]
|
|
236
|
-
when :closed
|
|
237
|
-
failures = state[:failures] + 1
|
|
238
|
-
@breaker_states[key] = if failures >= threshold
|
|
239
|
-
{
|
|
240
|
-
state: :open,
|
|
241
|
-
failures: failures,
|
|
242
|
-
last_failure: now,
|
|
243
|
-
opens_at: now + timeout
|
|
244
|
-
}
|
|
245
|
-
else
|
|
246
|
-
state.merge(failures: failures, last_failure: now)
|
|
247
|
-
end
|
|
248
|
-
when :half_open
|
|
249
|
-
@breaker_states[key] = {
|
|
250
|
-
state: :open,
|
|
251
|
-
failures: state[:failures],
|
|
252
|
-
last_failure: now,
|
|
253
|
-
opens_at: now + timeout
|
|
254
|
-
}
|
|
255
|
-
end
|
|
256
|
-
|
|
257
|
-
@breaker_states[key]
|
|
258
|
-
end
|
|
259
|
-
end
|
|
260
|
-
|
|
261
|
-
def trip_breaker(key, timeout)
|
|
262
|
-
with_write_lock("breaker:#{key}") do
|
|
263
|
-
now = current_time
|
|
264
|
-
@breaker_states[key] = {
|
|
265
|
-
state: :open,
|
|
266
|
-
failures: 0,
|
|
267
|
-
last_failure: now,
|
|
268
|
-
opens_at: now + timeout
|
|
269
|
-
}
|
|
270
|
-
end
|
|
271
|
-
end
|
|
272
|
-
|
|
273
|
-
def reset_breaker(key)
|
|
274
|
-
with_write_lock("breaker:#{key}") { @breaker_states.delete(key) }
|
|
275
|
-
end
|
|
182
|
+
# No circuit breaker operations here: breaker state is owned by BreakerMachines
|
|
276
183
|
|
|
277
184
|
# Utility operations
|
|
278
185
|
def clear(pattern = nil)
|
|
@@ -280,7 +187,7 @@ module ThrottleMachines
|
|
|
280
187
|
regex = Regexp.new(pattern.gsub('*', '.*'))
|
|
281
188
|
|
|
282
189
|
# Clear matching keys from all stores
|
|
283
|
-
[@counters, @gcra_states, @token_buckets
|
|
190
|
+
[@counters, @gcra_states, @token_buckets].each do |store|
|
|
284
191
|
store.each_key do |k|
|
|
285
192
|
store.delete(k) if k&.match?(regex)
|
|
286
193
|
end
|
|
@@ -289,7 +196,6 @@ module ThrottleMachines
|
|
|
289
196
|
@counters.clear
|
|
290
197
|
@gcra_states.clear
|
|
291
198
|
@token_buckets.clear
|
|
292
|
-
@breaker_states.clear
|
|
293
199
|
end
|
|
294
200
|
end
|
|
295
201
|
|
|
@@ -306,12 +212,12 @@ module ThrottleMachines
|
|
|
306
212
|
|
|
307
213
|
private
|
|
308
214
|
|
|
309
|
-
def with_read_lock(key, &)
|
|
310
|
-
lock_for(key).with_read_lock(&)
|
|
215
|
+
def with_read_lock(key, &block)
|
|
216
|
+
lock_for(key).with_read_lock(&block)
|
|
311
217
|
end
|
|
312
218
|
|
|
313
|
-
def with_write_lock(key, &)
|
|
314
|
-
lock_for(key).with_write_lock(&)
|
|
219
|
+
def with_write_lock(key, &block)
|
|
220
|
+
lock_for(key).with_write_lock(&block)
|
|
315
221
|
end
|
|
316
222
|
|
|
317
223
|
def lock_for(key)
|
|
@@ -351,20 +257,7 @@ module ThrottleMachines
|
|
|
351
257
|
with_write_lock(key) { @token_buckets.delete(key) } if data[:expires_at] && data[:expires_at] <= now
|
|
352
258
|
end
|
|
353
259
|
|
|
354
|
-
#
|
|
355
|
-
@breaker_states.each_pair do |key, data|
|
|
356
|
-
should_delete = false
|
|
357
|
-
|
|
358
|
-
# Clean closed states that have been idle
|
|
359
|
-
should_delete = true if data[:state] == :closed && data[:failures].zero?
|
|
360
|
-
|
|
361
|
-
# Clean expired open states (older than 2x timeout)
|
|
362
|
-
if data[:opens_at] && now > data[:opens_at] + ((data[:opens_at] - (data[:last_failure] || now)) * 2)
|
|
363
|
-
should_delete = true
|
|
364
|
-
end
|
|
365
|
-
|
|
366
|
-
with_write_lock("breaker:#{key}") { @breaker_states.delete(key) } if should_delete
|
|
367
|
-
end
|
|
260
|
+
# No breaker state cleanup: breaker state is managed by BreakerMachines
|
|
368
261
|
rescue StandardError => e
|
|
369
262
|
# Log error but don't crash cleanup thread
|
|
370
263
|
warn "ThrottleMachines: Cleanup error: #{e.message}"
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative 'base'
|
|
4
|
-
|
|
5
3
|
module ThrottleMachines
|
|
6
4
|
module Storage
|
|
7
5
|
class Null < Base
|
|
@@ -56,27 +54,6 @@ module ThrottleMachines
|
|
|
56
54
|
}
|
|
57
55
|
end
|
|
58
56
|
|
|
59
|
-
# Circuit breaker operations
|
|
60
|
-
def get_breaker_state(_key)
|
|
61
|
-
{ state: :closed, failures: 0, last_failure: nil }
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
def record_breaker_success(_key, _timeout, _half_open_requests = 1)
|
|
65
|
-
true
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
def record_breaker_failure(_key, _threshold, _timeout)
|
|
69
|
-
{ state: :closed, failures: 0, last_failure: nil }
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
def trip_breaker(_key, _timeout)
|
|
73
|
-
true
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
def reset_breaker(_key)
|
|
77
|
-
true
|
|
78
|
-
end
|
|
79
|
-
|
|
80
57
|
# Utility operations
|
|
81
58
|
def clear(_pattern = nil)
|
|
82
59
|
true
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
local key = KEYS[1]
|
|
2
|
+
local emission_interval = tonumber(ARGV[1])
|
|
3
|
+
local delay_tolerance = tonumber(ARGV[2])
|
|
4
|
+
local ttl = tonumber(ARGV[3])
|
|
5
|
+
local now = tonumber(ARGV[4])
|
|
6
|
+
|
|
7
|
+
local tat = redis.call('GET', key)
|
|
8
|
+
if not tat then
|
|
9
|
+
tat = 0
|
|
10
|
+
else
|
|
11
|
+
tat = tonumber(tat)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
tat = math.max(tat, now)
|
|
15
|
+
local allow = (tat - now) <= delay_tolerance
|
|
16
|
+
|
|
17
|
+
if allow then
|
|
18
|
+
local new_tat = tat + emission_interval
|
|
19
|
+
redis.call('SET', key, new_tat, 'EX', ttl)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
return { allow and 1 or 0, tat }
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
local key = KEYS[1]
|
|
2
|
+
local emission_interval = tonumber(ARGV[1])
|
|
3
|
+
local delay_tolerance = tonumber(ARGV[2])
|
|
4
|
+
local now = tonumber(ARGV[3])
|
|
5
|
+
|
|
6
|
+
local tat = redis.call('GET', key)
|
|
7
|
+
if not tat then
|
|
8
|
+
tat = 0
|
|
9
|
+
else
|
|
10
|
+
tat = tonumber(tat)
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
tat = math.max(tat, now)
|
|
14
|
+
local allow = (tat - now) <= delay_tolerance
|
|
15
|
+
|
|
16
|
+
return { allow and 1 or 0, tat }
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
local key = KEYS[1]
|
|
2
|
+
local capacity = tonumber(ARGV[1])
|
|
3
|
+
local refill_rate = tonumber(ARGV[2])
|
|
4
|
+
local now = tonumber(ARGV[3])
|
|
5
|
+
|
|
6
|
+
local bucket = redis.call('HMGET', key, 'tokens', 'last_refill')
|
|
7
|
+
local tokens = tonumber(bucket[1]) or capacity
|
|
8
|
+
local last_refill = tonumber(bucket[2]) or now
|
|
9
|
+
|
|
10
|
+
-- Calculate tokens without modifying
|
|
11
|
+
local elapsed = now - last_refill
|
|
12
|
+
local tokens_to_add = elapsed * refill_rate
|
|
13
|
+
tokens = math.min(tokens + tokens_to_add, capacity)
|
|
14
|
+
|
|
15
|
+
local allow = tokens >= 1
|
|
16
|
+
local tokens_after = allow and (tokens - 1) or 0
|
|
17
|
+
|
|
18
|
+
return { allow and 1 or 0, tokens_after }
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
local key = KEYS[1]
|
|
2
|
+
local capacity = tonumber(ARGV[1])
|
|
3
|
+
local refill_rate = tonumber(ARGV[2])
|
|
4
|
+
local ttl = tonumber(ARGV[3])
|
|
5
|
+
local now = tonumber(ARGV[4])
|
|
6
|
+
|
|
7
|
+
local bucket = redis.call('HMGET', key, 'tokens', 'last_refill')
|
|
8
|
+
local tokens = tonumber(bucket[1]) or capacity
|
|
9
|
+
local last_refill = tonumber(bucket[2]) or now
|
|
10
|
+
|
|
11
|
+
-- Refill tokens
|
|
12
|
+
local elapsed = now - last_refill
|
|
13
|
+
local tokens_to_add = elapsed * refill_rate
|
|
14
|
+
tokens = math.min(tokens + tokens_to_add, capacity)
|
|
15
|
+
|
|
16
|
+
local allow = tokens >= 1
|
|
17
|
+
if allow then
|
|
18
|
+
tokens = tokens - 1
|
|
19
|
+
redis.call('HMSET', key, 'tokens', tokens, 'last_refill', now)
|
|
20
|
+
redis.call('EXPIRE', key, ttl)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
return { allow and 1 or 0, tokens }
|
|
@@ -1,100 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative 'base'
|
|
4
|
-
|
|
5
3
|
module ThrottleMachines
|
|
6
4
|
module Storage
|
|
7
5
|
class Redis < Base
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
local emission_interval = tonumber(ARGV[1])
|
|
11
|
-
local delay_tolerance = tonumber(ARGV[2])
|
|
12
|
-
local ttl = tonumber(ARGV[3])
|
|
13
|
-
local now = tonumber(ARGV[4])
|
|
14
|
-
|
|
15
|
-
local tat = redis.call('GET', key)
|
|
16
|
-
if not tat then
|
|
17
|
-
tat = 0
|
|
18
|
-
else
|
|
19
|
-
tat = tonumber(tat)
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
tat = math.max(tat, now)
|
|
23
|
-
local allow = (tat - now) <= delay_tolerance
|
|
24
|
-
|
|
25
|
-
if allow then
|
|
26
|
-
local new_tat = tat + emission_interval
|
|
27
|
-
redis.call('SET', key, new_tat, 'EX', ttl)
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
return { allow and 1 or 0, tat }
|
|
31
|
-
LUA
|
|
32
|
-
|
|
33
|
-
TOKEN_BUCKET_SCRIPT = <<~LUA
|
|
34
|
-
local key = KEYS[1]
|
|
35
|
-
local capacity = tonumber(ARGV[1])
|
|
36
|
-
local refill_rate = tonumber(ARGV[2])
|
|
37
|
-
local ttl = tonumber(ARGV[3])
|
|
38
|
-
local now = tonumber(ARGV[4])
|
|
39
|
-
|
|
40
|
-
local bucket = redis.call('HMGET', key, 'tokens', 'last_refill')
|
|
41
|
-
local tokens = tonumber(bucket[1]) or capacity
|
|
42
|
-
local last_refill = tonumber(bucket[2]) or now
|
|
43
|
-
|
|
44
|
-
-- Refill tokens
|
|
45
|
-
local elapsed = now - last_refill
|
|
46
|
-
local tokens_to_add = elapsed * refill_rate
|
|
47
|
-
tokens = math.min(tokens + tokens_to_add, capacity)
|
|
48
|
-
|
|
49
|
-
local allow = tokens >= 1
|
|
50
|
-
if allow then
|
|
51
|
-
tokens = tokens - 1
|
|
52
|
-
redis.call('HMSET', key, 'tokens', tokens, 'last_refill', now)
|
|
53
|
-
redis.call('EXPIRE', key, ttl)
|
|
54
|
-
end
|
|
55
|
-
|
|
56
|
-
return { allow and 1 or 0, tokens }
|
|
57
|
-
LUA
|
|
58
|
-
|
|
59
|
-
PEEK_GCRA_SCRIPT = <<~LUA
|
|
60
|
-
local key = KEYS[1]
|
|
61
|
-
local emission_interval = tonumber(ARGV[1])
|
|
62
|
-
local delay_tolerance = tonumber(ARGV[2])
|
|
63
|
-
local now = tonumber(ARGV[3])
|
|
64
|
-
|
|
65
|
-
local tat = redis.call('GET', key)
|
|
66
|
-
if not tat then
|
|
67
|
-
tat = 0
|
|
68
|
-
else
|
|
69
|
-
tat = tonumber(tat)
|
|
70
|
-
end
|
|
6
|
+
# Load Lua scripts from files
|
|
7
|
+
LUA_SCRIPTS_DIR = File.expand_path('redis', __dir__)
|
|
71
8
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
PEEK_TOKEN_BUCKET_SCRIPT = <<~LUA
|
|
79
|
-
local key = KEYS[1]
|
|
80
|
-
local capacity = tonumber(ARGV[1])
|
|
81
|
-
local refill_rate = tonumber(ARGV[2])
|
|
82
|
-
local now = tonumber(ARGV[3])
|
|
83
|
-
|
|
84
|
-
local bucket = redis.call('HMGET', key, 'tokens', 'last_refill')
|
|
85
|
-
local tokens = tonumber(bucket[1]) or capacity
|
|
86
|
-
local last_refill = tonumber(bucket[2]) or now
|
|
87
|
-
|
|
88
|
-
-- Calculate tokens without modifying
|
|
89
|
-
local elapsed = now - last_refill
|
|
90
|
-
local tokens_to_add = elapsed * refill_rate
|
|
91
|
-
tokens = math.min(tokens + tokens_to_add, capacity)
|
|
92
|
-
|
|
93
|
-
local allow = tokens >= 1
|
|
94
|
-
local tokens_after = allow and (tokens - 1) or 0
|
|
95
|
-
|
|
96
|
-
return { allow and 1 or 0, tokens_after }
|
|
97
|
-
LUA
|
|
9
|
+
GCRA_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'gcra.lua'))
|
|
10
|
+
TOKEN_BUCKET_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'token_bucket.lua'))
|
|
11
|
+
PEEK_GCRA_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'peek_gcra.lua'))
|
|
12
|
+
PEEK_TOKEN_BUCKET_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'peek_token_bucket.lua'))
|
|
13
|
+
INCREMENT_COUNTER_SCRIPT = File.read(File.join(LUA_SCRIPTS_DIR, 'increment_counter.lua'))
|
|
14
|
+
# Breaker scripts removed: breaker state is owned by BreakerMachines
|
|
98
15
|
|
|
99
16
|
def initialize(options = {})
|
|
100
17
|
super
|
|
@@ -117,17 +34,7 @@ module ThrottleMachines
|
|
|
117
34
|
|
|
118
35
|
# Use Lua script for atomic increment with TTL
|
|
119
36
|
with_redis do |redis|
|
|
120
|
-
redis.eval(
|
|
121
|
-
local count = redis.call('INCRBY', KEYS[1], ARGV[1])
|
|
122
|
-
local ttl = redis.call('TTL', KEYS[1])
|
|
123
|
-
|
|
124
|
-
-- Set expiry if key is new (ttl == -2) or has no TTL (ttl == -1)
|
|
125
|
-
if ttl <= 0 then
|
|
126
|
-
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
|
127
|
-
end
|
|
128
|
-
|
|
129
|
-
return count
|
|
130
|
-
LUA
|
|
37
|
+
redis.eval(INCREMENT_COUNTER_SCRIPT, keys: [window_key], argv: [amount, window.to_i])
|
|
131
38
|
end
|
|
132
39
|
end
|
|
133
40
|
|
|
@@ -256,134 +163,7 @@ module ThrottleMachines
|
|
|
256
163
|
retry
|
|
257
164
|
end
|
|
258
165
|
|
|
259
|
-
#
|
|
260
|
-
def get_breaker_state(key)
|
|
261
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
262
|
-
|
|
263
|
-
# Use Lua script for atomic read and potential state transition
|
|
264
|
-
result = with_redis do |redis|
|
|
265
|
-
redis.eval(<<~LUA, keys: [breaker_key], argv: [current_time])
|
|
266
|
-
local data = redis.call('HGETALL', KEYS[1])
|
|
267
|
-
if #data == 0 then
|
|
268
|
-
return {}
|
|
269
|
-
end
|
|
270
|
-
|
|
271
|
-
local state = {}
|
|
272
|
-
for i = 1, #data, 2 do
|
|
273
|
-
state[data[i]] = data[i + 1]
|
|
274
|
-
end
|
|
275
|
-
|
|
276
|
-
-- Auto-transition from open to half-open if timeout passed
|
|
277
|
-
if state['state'] == 'open' and state['opens_at'] then
|
|
278
|
-
local now = tonumber(ARGV[1])
|
|
279
|
-
local opens_at = tonumber(state['opens_at'])
|
|
280
|
-
#{' '}
|
|
281
|
-
if now >= opens_at then
|
|
282
|
-
redis.call('HSET', KEYS[1], 'state', 'half_open', 'half_open_attempts', '0')
|
|
283
|
-
state['state'] = 'half_open'
|
|
284
|
-
state['half_open_attempts'] = '0'
|
|
285
|
-
end
|
|
286
|
-
end
|
|
287
|
-
|
|
288
|
-
return state
|
|
289
|
-
LUA
|
|
290
|
-
end
|
|
291
|
-
|
|
292
|
-
return { state: :closed, failures: 0, last_failure: nil } if result.empty?
|
|
293
|
-
|
|
294
|
-
# Convert hash from Lua to Ruby format
|
|
295
|
-
state = {}
|
|
296
|
-
result.each_slice(2) { |k, v| state[k] = v }
|
|
297
|
-
|
|
298
|
-
{
|
|
299
|
-
state: state['state'].to_sym,
|
|
300
|
-
failures: state['failures'].to_i,
|
|
301
|
-
last_failure: state['last_failure']&.to_f,
|
|
302
|
-
opens_at: state['opens_at']&.to_f,
|
|
303
|
-
half_open_attempts: state['half_open_attempts']&.to_i
|
|
304
|
-
}
|
|
305
|
-
end
|
|
306
|
-
|
|
307
|
-
def record_breaker_success(key, _timeout, half_open_requests = 1)
|
|
308
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
309
|
-
|
|
310
|
-
# Use Lua script for atomic success recording
|
|
311
|
-
with_redis do |redis|
|
|
312
|
-
redis.eval(<<~LUA, keys: [breaker_key], argv: [half_open_requests])
|
|
313
|
-
local state = redis.call('HGET', KEYS[1], 'state')
|
|
314
|
-
|
|
315
|
-
if state == 'half_open' then
|
|
316
|
-
-- Increment half-open attempts and potentially close the circuit
|
|
317
|
-
local attempts = redis.call('HINCRBY', KEYS[1], 'half_open_attempts', 1)
|
|
318
|
-
#{' '}
|
|
319
|
-
if attempts >= tonumber(ARGV[1]) then
|
|
320
|
-
redis.call('DEL', KEYS[1])
|
|
321
|
-
end
|
|
322
|
-
elseif state == 'closed' then
|
|
323
|
-
-- Reset failure count on success in closed state
|
|
324
|
-
local failures = redis.call('HGET', KEYS[1], 'failures')
|
|
325
|
-
if failures and tonumber(failures) > 0 then
|
|
326
|
-
redis.call('HSET', KEYS[1], 'failures', 0)
|
|
327
|
-
end
|
|
328
|
-
end
|
|
329
|
-
LUA
|
|
330
|
-
end
|
|
331
|
-
end
|
|
332
|
-
|
|
333
|
-
def record_breaker_failure(key, threshold, timeout)
|
|
334
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
335
|
-
now = current_time
|
|
336
|
-
|
|
337
|
-
# Use Lua script for atomic failure recording
|
|
338
|
-
with_redis do |redis|
|
|
339
|
-
redis.eval(<<~LUA, keys: [breaker_key], argv: [threshold, timeout, now])
|
|
340
|
-
local state = redis.call('HGET', KEYS[1], 'state') or 'closed'
|
|
341
|
-
local now = ARGV[3]
|
|
342
|
-
local timeout = tonumber(ARGV[2])
|
|
343
|
-
|
|
344
|
-
if state == 'half_open' then
|
|
345
|
-
-- Failure in half-open state, just re-open the circuit
|
|
346
|
-
redis.call('HMSET', KEYS[1],
|
|
347
|
-
'state', 'open',
|
|
348
|
-
'opens_at', tonumber(now) + timeout,
|
|
349
|
-
'last_failure', now
|
|
350
|
-
)
|
|
351
|
-
else -- state is 'closed' or nil
|
|
352
|
-
local failures = redis.call('HINCRBY', KEYS[1], 'failures', 1)
|
|
353
|
-
redis.call('HSET', KEYS[1], 'last_failure', now)
|
|
354
|
-
#{' '}
|
|
355
|
-
if failures >= tonumber(ARGV[1]) then
|
|
356
|
-
redis.call('HMSET', KEYS[1],
|
|
357
|
-
'state', 'open',
|
|
358
|
-
'opens_at', tonumber(now) + timeout
|
|
359
|
-
)
|
|
360
|
-
end
|
|
361
|
-
end
|
|
362
|
-
|
|
363
|
-
redis.call('EXPIRE', KEYS[1], timeout * 2)
|
|
364
|
-
LUA
|
|
365
|
-
end
|
|
366
|
-
|
|
367
|
-
get_breaker_state(key)
|
|
368
|
-
end
|
|
369
|
-
|
|
370
|
-
def trip_breaker(key, timeout)
|
|
371
|
-
breaker_key = prefixed("breaker:#{key}")
|
|
372
|
-
now = current_time
|
|
373
|
-
|
|
374
|
-
with_redis do |redis|
|
|
375
|
-
redis.hmset(breaker_key,
|
|
376
|
-
'state', 'open',
|
|
377
|
-
'failures', 0,
|
|
378
|
-
'last_failure', now,
|
|
379
|
-
'opens_at', now + timeout)
|
|
380
|
-
redis.expire(breaker_key, (timeout * 2).to_i)
|
|
381
|
-
end
|
|
382
|
-
end
|
|
383
|
-
|
|
384
|
-
def reset_breaker(key)
|
|
385
|
-
with_redis { |r| r.del(prefixed("breaker:#{key}")) }
|
|
386
|
-
end
|
|
166
|
+
# No circuit breaker operations: breaker state is owned by BreakerMachines
|
|
387
167
|
|
|
388
168
|
# Utility operations
|
|
389
169
|
def clear(pattern = nil)
|
|
@@ -437,13 +217,13 @@ module ThrottleMachines
|
|
|
437
217
|
raise ArgumentError, "Invalid Redis connection: #{e.message}"
|
|
438
218
|
end
|
|
439
219
|
|
|
440
|
-
def with_redis(&)
|
|
220
|
+
def with_redis(&block)
|
|
441
221
|
if @redis.respond_to?(:with)
|
|
442
222
|
# Connection pool
|
|
443
|
-
@redis.with(&)
|
|
223
|
+
@redis.with(&block)
|
|
444
224
|
else
|
|
445
225
|
# Regular Redis client
|
|
446
|
-
|
|
226
|
+
block.call(@redis)
|
|
447
227
|
end
|
|
448
228
|
end
|
|
449
229
|
end
|
data/lib/throttle_machines.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'json'
|
|
4
4
|
require 'timeout'
|
|
5
5
|
require 'zeitwerk'
|
|
6
|
-
require 'active_support/
|
|
6
|
+
require 'active_support/core_ext/class/attribute'
|
|
7
7
|
|
|
8
8
|
# Ecosystem dependencies
|
|
9
9
|
require 'chrono_machines'
|
|
@@ -14,25 +14,32 @@ loader = Zeitwerk::Loader.for_gem
|
|
|
14
14
|
loader.ignore("#{__dir__}/throttle_machines/engine.rb") unless defined?(Rails::Engine)
|
|
15
15
|
loader.setup
|
|
16
16
|
|
|
17
|
-
# Load conditional dependencies manually
|
|
18
|
-
require_relative 'throttle_machines/storage/redis' if defined?(Redis)
|
|
19
|
-
|
|
20
17
|
module ThrottleMachines
|
|
21
|
-
|
|
18
|
+
class Configuration
|
|
19
|
+
attr_accessor :default_limit, :default_period, :default_storage, :clock,
|
|
20
|
+
:instrumentation_enabled, :instrumentation_backend, :_storage_instance
|
|
21
|
+
|
|
22
|
+
def initialize
|
|
23
|
+
@default_limit = 100
|
|
24
|
+
@default_period = 60 # 1 minute
|
|
25
|
+
@default_storage = :memory
|
|
26
|
+
@clock = nil
|
|
27
|
+
@instrumentation_enabled = true
|
|
28
|
+
@instrumentation_backend = nil
|
|
29
|
+
@_storage_instance = nil
|
|
30
|
+
end
|
|
31
|
+
end
|
|
22
32
|
|
|
23
|
-
|
|
24
|
-
config_accessor :default_limit, default: 100
|
|
25
|
-
config_accessor :default_period, default: 60 # 1 minute
|
|
26
|
-
config_accessor :default_storage, default: :memory
|
|
27
|
-
config_accessor :clock, default: nil
|
|
28
|
-
config_accessor :instrumentation_enabled, default: true
|
|
29
|
-
config_accessor :instrumentation_backend
|
|
30
|
-
config_accessor :_storage_instance
|
|
33
|
+
@config = Configuration.new
|
|
31
34
|
|
|
32
35
|
class << self
|
|
33
36
|
# Delegate monotonic time to BreakerMachines for consistency
|
|
34
37
|
delegate :monotonic_time, to: :BreakerMachines
|
|
35
38
|
|
|
39
|
+
def config
|
|
40
|
+
@config
|
|
41
|
+
end
|
|
42
|
+
|
|
36
43
|
def configure
|
|
37
44
|
yield(config) if block_given?
|
|
38
45
|
|
|
@@ -68,28 +75,28 @@ module ThrottleMachines
|
|
|
68
75
|
control
|
|
69
76
|
end
|
|
70
77
|
|
|
71
|
-
def limit(key, limit:, period:, algorithm: :fixed_window, &)
|
|
78
|
+
def limit(key, limit:, period:, algorithm: :fixed_window, &block)
|
|
72
79
|
limiter = limiter(key, limit: limit, period: period, algorithm: algorithm)
|
|
73
|
-
limiter.throttle!(&)
|
|
80
|
+
limiter.throttle!(&block)
|
|
74
81
|
end
|
|
75
82
|
|
|
76
|
-
def break_circuit(key, failures:, timeout:, &)
|
|
77
|
-
# Delegate to
|
|
83
|
+
def break_circuit(key, failures:, timeout:, &block)
|
|
84
|
+
# Delegate to BreakerMachines; use reset_timeout for open duration
|
|
78
85
|
breaker = BreakerMachines::Circuit.new(
|
|
79
|
-
key
|
|
86
|
+
key,
|
|
80
87
|
failure_threshold: failures,
|
|
81
|
-
|
|
88
|
+
reset_timeout: timeout
|
|
82
89
|
)
|
|
83
|
-
breaker.call(&)
|
|
90
|
+
breaker.call(&block)
|
|
84
91
|
end
|
|
85
92
|
|
|
86
|
-
def retry_with(max_attempts: 3, backoff: :exponential, &)
|
|
93
|
+
def retry_with(max_attempts: 3, backoff: :exponential, &block)
|
|
87
94
|
# Delegate to chrono_machines
|
|
88
95
|
policy_options = {
|
|
89
96
|
max_attempts: max_attempts,
|
|
90
97
|
jitter_factor: backoff == :exponential ? 1.0 : 0.0
|
|
91
98
|
}
|
|
92
|
-
ChronoMachines.retry(policy_options, &)
|
|
99
|
+
ChronoMachines.retry(policy_options, &block)
|
|
93
100
|
end
|
|
94
101
|
|
|
95
102
|
def limiter(key, limit:, period:, algorithm: :fixed_window)
|
|
@@ -132,7 +139,9 @@ module ThrottleMachines
|
|
|
132
139
|
# Auto-configure with defaults
|
|
133
140
|
configure
|
|
134
141
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
142
|
+
CircuitOpenError = BreakerMachines::CircuitOpenError
|
|
143
|
+
RetryExhaustedError = ChronoMachines::MaxRetriesExceededError
|
|
144
|
+
|
|
145
|
+
# Back-compat wrapper: use BreakerMachines as the circuit implementation
|
|
146
|
+
Breaker = BreakerMachines::Circuit
|
|
138
147
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: throttle_machines
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Abdelkader Boudih
|
|
@@ -15,14 +15,14 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version:
|
|
18
|
+
version: 8.0.4
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - ">="
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version:
|
|
25
|
+
version: 8.0.4
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
27
|
name: concurrent-ruby
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -71,14 +71,14 @@ dependencies:
|
|
|
71
71
|
requirements:
|
|
72
72
|
- - "~>"
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
|
-
version: '0.
|
|
74
|
+
version: '0.7'
|
|
75
75
|
type: :runtime
|
|
76
76
|
prerelease: false
|
|
77
77
|
version_requirements: !ruby/object:Gem::Requirement
|
|
78
78
|
requirements:
|
|
79
79
|
- - "~>"
|
|
80
80
|
- !ruby/object:Gem::Version
|
|
81
|
-
version: '0.
|
|
81
|
+
version: '0.7'
|
|
82
82
|
- !ruby/object:Gem::Dependency
|
|
83
83
|
name: chrono_machines
|
|
84
84
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -101,16 +101,16 @@ executables: []
|
|
|
101
101
|
extensions: []
|
|
102
102
|
extra_rdoc_files: []
|
|
103
103
|
files:
|
|
104
|
-
-
|
|
104
|
+
- LICENSE
|
|
105
105
|
- README.md
|
|
106
106
|
- Rakefile
|
|
107
107
|
- lib/throttle_machines.rb
|
|
108
108
|
- lib/throttle_machines/async_limiter.rb
|
|
109
|
-
- lib/throttle_machines/clock.rb
|
|
110
109
|
- lib/throttle_machines/control.rb
|
|
111
110
|
- lib/throttle_machines/controller_helpers.rb
|
|
112
111
|
- lib/throttle_machines/dependency_error.rb
|
|
113
112
|
- lib/throttle_machines/engine.rb
|
|
113
|
+
- lib/throttle_machines/hedged_breaker.rb
|
|
114
114
|
- lib/throttle_machines/hedged_request.rb
|
|
115
115
|
- lib/throttle_machines/instrumentation.rb
|
|
116
116
|
- lib/throttle_machines/limiter.rb
|
|
@@ -128,6 +128,11 @@ files:
|
|
|
128
128
|
- lib/throttle_machines/storage/memory.rb
|
|
129
129
|
- lib/throttle_machines/storage/null.rb
|
|
130
130
|
- lib/throttle_machines/storage/redis.rb
|
|
131
|
+
- lib/throttle_machines/storage/redis/gcra.lua
|
|
132
|
+
- lib/throttle_machines/storage/redis/increment_counter.lua
|
|
133
|
+
- lib/throttle_machines/storage/redis/peek_gcra.lua
|
|
134
|
+
- lib/throttle_machines/storage/redis/peek_token_bucket.lua
|
|
135
|
+
- lib/throttle_machines/storage/redis/token_bucket.lua
|
|
131
136
|
- lib/throttle_machines/throttled_error.rb
|
|
132
137
|
- lib/throttle_machines/version.rb
|
|
133
138
|
homepage: https://github.com/seuros/throttle_machines
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module ThrottleMachines
|
|
4
|
-
# TestClock for time manipulation in tests
|
|
5
|
-
# This overrides the global monotonic_time method for testing
|
|
6
|
-
class TestClock
|
|
7
|
-
attr_accessor :current_time
|
|
8
|
-
|
|
9
|
-
def initialize(start_time = Time.now.to_f)
|
|
10
|
-
@current_time = start_time
|
|
11
|
-
|
|
12
|
-
# Override the global monotonic_time method
|
|
13
|
-
ThrottleMachines.singleton_class.define_method(:monotonic_time) do
|
|
14
|
-
@current_time
|
|
15
|
-
end
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def now
|
|
19
|
-
@current_time
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def monotonic
|
|
23
|
-
@current_time
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
def advance(seconds)
|
|
27
|
-
@current_time += seconds
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def travel_to(time)
|
|
31
|
-
@current_time = time.to_f
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def reset
|
|
35
|
-
# Restore the original monotonic_time method
|
|
36
|
-
ThrottleMachines.singleton_class.define_method(:monotonic_time) do
|
|
37
|
-
BreakerMachines.monotonic_time
|
|
38
|
-
end
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
end
|
/data/{MIT-LICENSE → LICENSE}
RENAMED
|
File without changes
|