robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Distributed rate limiting, usable from jobs or any other Ruby code.
|
|
5
|
+
#
|
|
6
|
+
# STRIPE = Robin::Limiter.concurrent("stripe", 10) # at most 10 at once
|
|
7
|
+
# GITHUB = Robin::Limiter.bucket("github", 5_000, :hour) # 5000 per calendar hour
|
|
8
|
+
# SMS = Robin::Limiter.window("sms", 10, :minute) # 10 in any 60s window
|
|
9
|
+
# EMAIL = Robin::Limiter.leaky("email", 60, 60) # drains 1/s, bursts to 60
|
|
10
|
+
# OPENAI = Robin::Limiter.points("openai", 90_000, 60) # token budget per minute
|
|
11
|
+
#
|
|
12
|
+
# STRIPE.within_limit { charge! }
|
|
13
|
+
# OPENAI.within_limit(points: tokens) { complete! }
|
|
14
|
+
# GITHUB.within_limit(used: 3) { sync_three_repos! } # bucket and window
|
|
15
|
+
#
|
|
16
|
+
# Limiters keep their state in the configured Redis; Limiter.via sends a
|
|
17
|
+
# thread's limiter calls to another one (e.g. a shard of their own):
|
|
18
|
+
#
|
|
19
|
+
# Robin::Limiter.via(LIMITER_REDIS) { STRIPE.within_limit { charge! } }
|
|
20
|
+
#
|
|
21
|
+
# The web UI lists the limiters in the configured Redis only.
|
|
22
|
+
#
|
|
23
|
+
# #within_limit waits up to +wait_timeout+ seconds for capacity and
|
|
24
|
+
# otherwise raises Robin::Limiter::OverLimit. Inside a job, Robin catches
|
|
25
|
+
# OverLimit and reschedules the job with a backoff; that doesn't count as
|
|
26
|
+
# a failure or use up retries.
|
|
27
|
+
#
|
|
28
|
+
# Every kind of limiter also takes:
|
|
29
|
+
#
|
|
30
|
+
# reschedule:: how many times a job that hits this limiter is rescheduled
|
|
31
|
+
# before the OverLimit counts as a failure (default
|
|
32
|
+
# config.max_rate_limit_retries; 0 fails at once)
|
|
33
|
+
# backoff:: a callable (count, error, payload) returning the seconds to
|
|
34
|
+
# wait before running the job again, or nil for the default
|
|
35
|
+
# (see also config.rate_limit_backoff)
|
|
36
|
+
# policy:: what #within_limit does when it can't get capacity in
|
|
37
|
+
# time: :raise (the default) raises OverLimit; :skip
|
|
38
|
+
# returns nil without running the block, for work that
|
|
39
|
+
# can be dropped (e.g. refreshing a cache)
|
|
40
|
+
#
|
|
41
|
+
# Robin::Limiter.unlimited never limits: swap it in where a limit
|
|
42
|
+
# shouldn't apply, e.g. in tests.
|
|
43
|
+
module Limiter
|
|
44
|
+
class OverLimit < Robin::Error
|
|
45
|
+
attr_reader :limiter, :retry_after
|
|
46
|
+
|
|
47
|
+
def initialize(limiter, retry_after = nil)
|
|
48
|
+
@limiter = limiter
|
|
49
|
+
@retry_after = retry_after
|
|
50
|
+
super("rate limiter #{limiter.name.inspect} (#{limiter.type}) is over its limit")
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# The end of every acquiring script (see Limiter.acquiring).
|
|
55
|
+
COUNT_OVER = <<~LUA
|
|
56
|
+
if taken == false or (type(taken) == 'number' and taken > 0) then
|
|
57
|
+
if (taken or tonumber(ARGV[#ARGV - 1])) > tonumber(ARGV[#ARGV]) then
|
|
58
|
+
redis.call('HINCRBY', KEYS[2], 'over', 1)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
return taken
|
|
62
|
+
LUA
|
|
63
|
+
|
|
64
|
+
# A limiter's acquiring script, from +body+: Lua that takes capacity
|
|
65
|
+
# (KEYS[1] its state, KEYS[2] its definition) and returns 0, or a
|
|
66
|
+
# token, when it took some, else how many ms until some frees up, or
|
|
67
|
+
# false when it can't tell. The script also counts the limiter over
|
|
68
|
+
# its limit when the wait is longer than the caller will wait. Its
|
|
69
|
+
# last two ARGV, after the body's own: the wait (ms) to assume when
|
|
70
|
+
# the body can't tell, and how long (ms) the caller will wait.
|
|
71
|
+
def self.acquiring(body) = Redis::Script.new("local taken = (function()\n#{body}\nend)()\n#{COUNT_OVER}")
|
|
72
|
+
|
|
73
|
+
INTERVALS = {second: 1, minute: 60, hour: 3_600, day: 86_400}.freeze
|
|
74
|
+
TYPES = {"concurrent" => "Concurrent", "bucket" => "Bucket", "window" => "Window",
|
|
75
|
+
"leaky" => "Leaky", "points" => "Points"}.freeze
|
|
76
|
+
META_TTL = 30 * 86_400
|
|
77
|
+
REGISTER_EVERY = 600
|
|
78
|
+
|
|
79
|
+
# The definition each limiter name last wrote to each Redis, and when:
|
|
80
|
+
# {[pool id, meta key] => [[type, limit, interval], monotonic time]}.
|
|
81
|
+
# Shared by every limiter object, so one built inside perform doesn't
|
|
82
|
+
# write its definition on every call.
|
|
83
|
+
@registered = {}
|
|
84
|
+
|
|
85
|
+
class << self
|
|
86
|
+
attr_reader :registered # :nodoc:
|
|
87
|
+
|
|
88
|
+
# Sends every limiter call this thread makes inside the block to
|
|
89
|
+
# +pool+ (a Robin::Redis::Pool) instead of the configured Redis.
|
|
90
|
+
def via(pool)
|
|
91
|
+
previous = Thread.current[:robin_limiter_pool]
|
|
92
|
+
Thread.current[:robin_limiter_pool] = pool
|
|
93
|
+
yield
|
|
94
|
+
ensure
|
|
95
|
+
Thread.current[:robin_limiter_pool] = previous
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# The pool set by an enclosing Limiter.via block, if any.
|
|
99
|
+
def current_pool = Thread.current[:robin_limiter_pool]
|
|
100
|
+
|
|
101
|
+
def concurrent(name, size, wait_timeout: 5, lock_timeout: 30, **)
|
|
102
|
+
Concurrent.new(name, size, wait_timeout: wait_timeout, lock_timeout: lock_timeout, **)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def bucket(name, count, interval, **) = Bucket.new(name, count, interval, **)
|
|
106
|
+
|
|
107
|
+
def window(name, count, interval, **) = Window.new(name, count, interval, **)
|
|
108
|
+
|
|
109
|
+
def leaky(name, size, interval, **) = Leaky.new(name, size, interval, **)
|
|
110
|
+
|
|
111
|
+
def points(name, points, interval, **) = Points.new(name, points, interval, **)
|
|
112
|
+
|
|
113
|
+
def unlimited(name = "unlimited") = Unlimited.new(name)
|
|
114
|
+
|
|
115
|
+
# Seconds for a :second/:minute/:hour/:day symbol (or string), a
|
|
116
|
+
# number of seconds, or nil.
|
|
117
|
+
def seconds(interval)
|
|
118
|
+
case interval
|
|
119
|
+
when Symbol, String then INTERVALS.fetch(interval.to_sym) { raise ArgumentError, "unknown interval #{interval.inspect}" }
|
|
120
|
+
when nil then nil
|
|
121
|
+
else Float(interval)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Every limiter used in the last 30 days, with current usage, in
|
|
126
|
+
# two round trips.
|
|
127
|
+
def list(config: Robin.config)
|
|
128
|
+
metas = Scripts.member_hashes(config, config.key("limiters"), "#{config.key("lmtr")}:")
|
|
129
|
+
expired, live = metas.partition { |_name, meta| meta.empty? }
|
|
130
|
+
limiters = live.map { |name, meta| from_meta(name, meta, config) }
|
|
131
|
+
usage = config.redis do |c|
|
|
132
|
+
c.pipelined do |p|
|
|
133
|
+
limiters.compact.each { |limiter| p.call(*limiter.used_command) }
|
|
134
|
+
p.call("SREM", config.key("limiters"), *expired.map(&:first)) unless expired.empty?
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
live.zip(limiters).map do |(name, meta), limiter|
|
|
138
|
+
{
|
|
139
|
+
"name" => name, "type" => meta.fetch("type"), "limit" => Integer(meta.fetch("limit")),
|
|
140
|
+
"interval" => interval_from(meta), "used" => limiter ? limiter.used_from(usage.shift) : 0,
|
|
141
|
+
"over_limit_count" => meta["over"].to_i, "last_used_at" => meta["last"]&.to_i
|
|
142
|
+
}
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Clears a limiter's state and over-limit counter, in two round trips.
|
|
147
|
+
def reset!(name, config: Robin.config)
|
|
148
|
+
clear(name, config) { |p, known| p.call("HSET", config.key("lmtr", name), "over", 0) if known }
|
|
149
|
+
true
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Forgets a limiter: its state and its definition, in two round trips.
|
|
153
|
+
def delete!(name, config: Robin.config)
|
|
154
|
+
clear(name, config) do |p|
|
|
155
|
+
p.call("DEL", config.key("lmtr", name))
|
|
156
|
+
p.call("SREM", config.key("limiters"), name)
|
|
157
|
+
end
|
|
158
|
+
@registered.delete([config.redis_pool.object_id, config.key("lmtr", name)])
|
|
159
|
+
true
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
private
|
|
163
|
+
|
|
164
|
+
# Clears limiter +name+'s state, along with what the block queues
|
|
165
|
+
# on the same pipeline; it's told whether the limiter has a
|
|
166
|
+
# definition (one never used, or expired, has no state to clear).
|
|
167
|
+
def clear(name, config)
|
|
168
|
+
meta = config.redis { |c| c.call("HGETALL", config.key("lmtr", name)) }.each_slice(2).to_h
|
|
169
|
+
limiter = from_meta(name, meta, config) unless meta.empty?
|
|
170
|
+
config.redis do |c|
|
|
171
|
+
c.pipelined do |p|
|
|
172
|
+
p.call(*limiter.reset_command) if limiter
|
|
173
|
+
yield p, !meta.empty?
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def from_meta(name, meta, config)
|
|
179
|
+
klass = TYPES[meta.fetch("type")]
|
|
180
|
+
klass && const_get(klass).new(name, meta.fetch("limit"), interval_from(meta), config: config)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def interval_from(meta)
|
|
184
|
+
raw = meta.fetch("interval")
|
|
185
|
+
Float(raw) unless raw.empty?
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
class Base
|
|
190
|
+
POLICIES = %i[raise skip].freeze
|
|
191
|
+
|
|
192
|
+
attr_reader :name, :limit, :interval, :interval_ms, :wait_timeout, :reschedule, :backoff, :policy
|
|
193
|
+
|
|
194
|
+
def initialize(name, limit, interval, wait_timeout: 5, reschedule: nil, backoff: nil, policy: :raise, config: nil)
|
|
195
|
+
@name = name.to_s
|
|
196
|
+
raise ArgumentError, "limiter name can't be blank" if @name.empty?
|
|
197
|
+
|
|
198
|
+
@limit = Integer(limit)
|
|
199
|
+
raise ArgumentError, "limit must be positive" unless @limit.positive?
|
|
200
|
+
|
|
201
|
+
@interval = seconds = Limiter.seconds(interval)
|
|
202
|
+
@interval_ms = Integer(seconds * 1000) if seconds
|
|
203
|
+
@wait_timeout = Float(wait_timeout)
|
|
204
|
+
@reschedule = reschedule && Integer(reschedule)
|
|
205
|
+
raise ArgumentError, "reschedule can't be negative" if @reschedule&.negative?
|
|
206
|
+
raise ArgumentError, "backoff must respond to #call" unless backoff.nil? || backoff.respond_to?(:call)
|
|
207
|
+
|
|
208
|
+
raise ArgumentError, "policy must be :raise or :skip, not #{policy.inspect}" unless POLICIES.include?(policy)
|
|
209
|
+
|
|
210
|
+
@backoff = backoff
|
|
211
|
+
@policy = policy
|
|
212
|
+
@config = config
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def config = @config || Robin.config
|
|
216
|
+
|
|
217
|
+
def type = self.class.name.split("::").last.downcase
|
|
218
|
+
|
|
219
|
+
# Runs the block once capacity is available. If it doesn't become
|
|
220
|
+
# available within +wait_timeout+, raises OverLimit or (with policy:
|
|
221
|
+
# :skip) returns nil without running the block.
|
|
222
|
+
def within_limit(**opts)
|
|
223
|
+
register
|
|
224
|
+
token = wait_for_capacity(opts) or return
|
|
225
|
+
begin
|
|
226
|
+
yield self
|
|
227
|
+
ensure
|
|
228
|
+
release(token)
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def over_limit!
|
|
233
|
+
redis { |c| c.call("HINCRBY", meta_key, "over", 1) }
|
|
234
|
+
rescue
|
|
235
|
+
nil
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def reset_state
|
|
239
|
+
redis { |c| c.call(*reset_command) }
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# The command #reset_state sends.
|
|
243
|
+
def reset_command = ["DEL", state_key]
|
|
244
|
+
|
|
245
|
+
# How many uses are counted now: what the reply to each kind's
|
|
246
|
+
# #used_command (Limiter.list sends them together) means.
|
|
247
|
+
def used = used_from(redis { |c| c.call(*used_command) })
|
|
248
|
+
|
|
249
|
+
def used_from(reply) = reply
|
|
250
|
+
|
|
251
|
+
def to_h = {"name" => name, "type" => type, "limit" => limit, "interval" => interval, "used" => used}
|
|
252
|
+
|
|
253
|
+
private
|
|
254
|
+
|
|
255
|
+
def meta_key = config.key("lmtr", name)
|
|
256
|
+
|
|
257
|
+
def state_key = config.key("lmtr", name, "s")
|
|
258
|
+
|
|
259
|
+
def now_ms = Robin.now_ms
|
|
260
|
+
|
|
261
|
+
def release(_token)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Scripts return 0 when they took capacity, else how many ms until
|
|
265
|
+
# some frees up. Returns [token, wait_ms].
|
|
266
|
+
def outcome(wait) = [wait.zero?, wait]
|
|
267
|
+
|
|
268
|
+
# Retries #acquire until it succeeds or waiting longer would pass the
|
|
269
|
+
# deadline. Limiters that know when capacity frees up say so (in ms);
|
|
270
|
+
# the others are polled with a short exponential backoff. Returns
|
|
271
|
+
# the token, or nil when skipping.
|
|
272
|
+
#
|
|
273
|
+
# The script counts the limiter over its limit when it gives up, in
|
|
274
|
+
# the same round trip: it's told the wait to assume (the backoff)
|
|
275
|
+
# and how long is left (+wait+, in ms).
|
|
276
|
+
def wait_for_capacity(opts)
|
|
277
|
+
deadline = monotonic + @wait_timeout
|
|
278
|
+
attempt = 0
|
|
279
|
+
loop do
|
|
280
|
+
guess = [0.01 * (2**attempt), 0.25].min * 1000
|
|
281
|
+
left = (deadline - monotonic) * 1000
|
|
282
|
+
token, wait_ms = acquire(**opts, wait: [guess, left])
|
|
283
|
+
return token if token
|
|
284
|
+
|
|
285
|
+
wait = wait_ms || guess
|
|
286
|
+
if wait > left
|
|
287
|
+
return if policy == :skip
|
|
288
|
+
|
|
289
|
+
raise OverLimit.new(self, wait_ms && wait_ms / 1000.0)
|
|
290
|
+
end
|
|
291
|
+
attempt += 1
|
|
292
|
+
sleep(wait / 1000.0 + rand * 0.005)
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Writes the definition for the web UI, once per process and Redis
|
|
297
|
+
# every 10 minutes, or as soon as a limiter of the same name with a
|
|
298
|
+
# different definition writes its own.
|
|
299
|
+
def register
|
|
300
|
+
target = pool
|
|
301
|
+
cache_key = [target.object_id, meta_key] #: [Integer, String]
|
|
302
|
+
definition = [type, limit, interval] #: [String, Integer, Integer | Float | nil]
|
|
303
|
+
written, at = Limiter.registered[cache_key]
|
|
304
|
+
return if written == definition && monotonic - at < REGISTER_EVERY
|
|
305
|
+
|
|
306
|
+
target.with do |c|
|
|
307
|
+
c.pipelined do |p|
|
|
308
|
+
p.call("HSET", meta_key, "type", type, "limit", limit, "interval", interval)
|
|
309
|
+
p.call("EXPIRE", meta_key, META_TTL)
|
|
310
|
+
p.call("SADD", config.key("limiters"), name)
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
Limiter.registered[cache_key] = [definition, monotonic]
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
317
|
+
|
|
318
|
+
def pool = Limiter.current_pool || config.redis_pool
|
|
319
|
+
|
|
320
|
+
def redis(&) = pool.with(&)
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# Lets one call count as several uses (+used:+), e.g. an API call that
|
|
324
|
+
# covers a batch of records.
|
|
325
|
+
module Uses
|
|
326
|
+
def within_limit(used: 1, &)
|
|
327
|
+
unless used.instance_of?(Integer) && used.between?(1, limit)
|
|
328
|
+
raise ArgumentError, "used (#{used.inspect}) must be a whole number from 1 to the limit (#{limit})"
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
super
|
|
332
|
+
end
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
# At most +size+ holders at once. Holders lease their slot for
|
|
336
|
+
# +lock_timeout+ seconds so a crashed process can't leak capacity.
|
|
337
|
+
class Concurrent < Base
|
|
338
|
+
# Returns the lease token, or false when full.
|
|
339
|
+
ACQUIRE = Limiter.acquiring(<<~LUA)
|
|
340
|
+
local now = tonumber(ARGV[1])
|
|
341
|
+
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
|
|
342
|
+
if redis.call('ZCARD', KEYS[1]) < tonumber(ARGV[2]) then
|
|
343
|
+
redis.call('ZADD', KEYS[1], now + tonumber(ARGV[3]), ARGV[4])
|
|
344
|
+
redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[3]) + 60000)
|
|
345
|
+
redis.call('HSET', KEYS[2], 'last', now)
|
|
346
|
+
return ARGV[4]
|
|
347
|
+
end
|
|
348
|
+
return false
|
|
349
|
+
LUA
|
|
350
|
+
|
|
351
|
+
attr_reader :lock_timeout, :lock_timeout_ms
|
|
352
|
+
|
|
353
|
+
def initialize(name, size, _interval = nil, lock_timeout: 30, **)
|
|
354
|
+
super(name, size, nil, **)
|
|
355
|
+
@lock_timeout = Float(lock_timeout)
|
|
356
|
+
@lock_timeout_ms = Integer(@lock_timeout * 1000)
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
def used_command = ["ZCOUNT", state_key, "(#{now_ms}", "+inf"]
|
|
360
|
+
|
|
361
|
+
private
|
|
362
|
+
|
|
363
|
+
# The lease token, or nil when full (no wait is announced: holders
|
|
364
|
+
# can release at any time, so callers poll).
|
|
365
|
+
def acquire(wait:, **)
|
|
366
|
+
redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, limit, lock_timeout_ms, Robin.generate_jid, *wait]) }
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
def release(token)
|
|
370
|
+
redis { |c| c.call("ZREM", state_key, token) }
|
|
371
|
+
end
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
# +limit+ uses per calendar interval (each second/minute/hour/day, or
|
|
375
|
+
# any number of seconds, aligned to the epoch).
|
|
376
|
+
class Bucket < Base
|
|
377
|
+
include Uses
|
|
378
|
+
|
|
379
|
+
# ARGV: limit, ttl (ms), now (ms), uses
|
|
380
|
+
ACQUIRE = Limiter.acquiring(<<~LUA)
|
|
381
|
+
local used = tonumber(ARGV[4])
|
|
382
|
+
local n = redis.call('INCRBY', KEYS[1], used)
|
|
383
|
+
if n == used then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
|
|
384
|
+
if n > tonumber(ARGV[1]) then
|
|
385
|
+
redis.call('DECRBY', KEYS[1], used)
|
|
386
|
+
return math.max(redis.call('PTTL', KEYS[1]), 1)
|
|
387
|
+
end
|
|
388
|
+
redis.call('HSET', KEYS[2], 'last', ARGV[3])
|
|
389
|
+
return 0
|
|
390
|
+
LUA
|
|
391
|
+
|
|
392
|
+
def initialize(name, limit, interval, **)
|
|
393
|
+
super
|
|
394
|
+
raise ArgumentError, "bucket limiters need an interval" unless interval_ms
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
def used_command = ["GET", bucket_key(now_ms)]
|
|
398
|
+
|
|
399
|
+
def used_from(reply) = reply.to_i
|
|
400
|
+
|
|
401
|
+
def reset_command = ["DEL", bucket_key(now_ms)]
|
|
402
|
+
|
|
403
|
+
private
|
|
404
|
+
|
|
405
|
+
def bucket_key(now) = config.key("lmtr", name, "b", now / interval_ms)
|
|
406
|
+
|
|
407
|
+
def acquire(used:, wait:)
|
|
408
|
+
now = now_ms
|
|
409
|
+
ttl = interval_ms - (now % interval_ms)
|
|
410
|
+
outcome(redis { |c| c.evalsha(ACQUIRE, [bucket_key(now), meta_key], [limit, ttl, now, used, *wait]) })
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# At most +limit+ uses in any rolling +interval+.
|
|
415
|
+
class Window < Base
|
|
416
|
+
include Uses
|
|
417
|
+
|
|
418
|
+
# ARGV: now (ms), interval (ms), limit, unique id, uses. Each use is
|
|
419
|
+
# a member; when they don't fit, the wait is until enough expire.
|
|
420
|
+
ACQUIRE = Limiter.acquiring(<<~LUA)
|
|
421
|
+
local now = tonumber(ARGV[1])
|
|
422
|
+
local interval = tonumber(ARGV[2])
|
|
423
|
+
local used = tonumber(ARGV[5])
|
|
424
|
+
redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now - interval)
|
|
425
|
+
local over = redis.call('ZCARD', KEYS[1]) + used - tonumber(ARGV[3])
|
|
426
|
+
if over <= 0 then
|
|
427
|
+
for i = 1, used do redis.call('ZADD', KEYS[1], now, ARGV[4] .. ':' .. i) end
|
|
428
|
+
redis.call('PEXPIRE', KEYS[1], interval + 1000)
|
|
429
|
+
redis.call('HSET', KEYS[2], 'last', now)
|
|
430
|
+
return 0
|
|
431
|
+
end
|
|
432
|
+
local freeing = redis.call('ZRANGE', KEYS[1], over - 1, over - 1, 'WITHSCORES')
|
|
433
|
+
return math.max(tonumber(freeing[2]) + interval - now, 1)
|
|
434
|
+
LUA
|
|
435
|
+
|
|
436
|
+
def initialize(name, limit, interval, **)
|
|
437
|
+
super
|
|
438
|
+
raise ArgumentError, "window limiters need an interval" unless interval_ms
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
def used_command = ["ZCOUNT", state_key, "(#{now_ms - interval_ms}", "+inf"]
|
|
442
|
+
|
|
443
|
+
private
|
|
444
|
+
|
|
445
|
+
def acquire(used:, wait:)
|
|
446
|
+
outcome(redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, interval_ms, limit, Robin.generate_jid, used, *wait]) })
|
|
447
|
+
end
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
# A bucket of capacity +limit+ that drains at limit/interval per second:
|
|
451
|
+
# allows bursts up to +limit+, then a steady rate.
|
|
452
|
+
class Leaky < Base
|
|
453
|
+
# Fractions are written with %.14g, as Redis's Lua 5.1 does, so a
|
|
454
|
+
# server with a newer Lua (Dragonfly writes 2 as "2.0") stores the
|
|
455
|
+
# same text.
|
|
456
|
+
ACQUIRE = Limiter.acquiring(<<~LUA)
|
|
457
|
+
local now = tonumber(ARGV[1])
|
|
458
|
+
local size = tonumber(ARGV[2])
|
|
459
|
+
local interval = tonumber(ARGV[3])
|
|
460
|
+
local level = tonumber(redis.call('HGET', KEYS[1], 'level') or '0')
|
|
461
|
+
local last = tonumber(redis.call('HGET', KEYS[1], 'at') or ARGV[1])
|
|
462
|
+
level = math.max(0, level - (now - last) * size / interval)
|
|
463
|
+
if level + 1 <= size then
|
|
464
|
+
redis.call('HSET', KEYS[1], 'level', string.format('%.14g', level + 1), 'at', now)
|
|
465
|
+
redis.call('PEXPIRE', KEYS[1], interval + 1000)
|
|
466
|
+
redis.call('HSET', KEYS[2], 'last', now)
|
|
467
|
+
return 0
|
|
468
|
+
end
|
|
469
|
+
redis.call('HSET', KEYS[1], 'level', string.format('%.14g', level), 'at', now)
|
|
470
|
+
return math.max(math.ceil((level + 1 - size) * interval / size), 1)
|
|
471
|
+
LUA
|
|
472
|
+
|
|
473
|
+
def initialize(name, limit, interval, **)
|
|
474
|
+
super
|
|
475
|
+
raise ArgumentError, "leaky limiters need an interval" unless interval_ms
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
# Current level; 0 when the limiter has never been used.
|
|
479
|
+
def used_command = ["HMGET", state_key, "level", "at"]
|
|
480
|
+
|
|
481
|
+
def used_from(reply)
|
|
482
|
+
level, at = reply
|
|
483
|
+
drained = (now_ms - at.to_i) * limit / interval_ms.to_f
|
|
484
|
+
[level.to_f - drained, 0].max.ceil
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
private
|
|
488
|
+
|
|
489
|
+
def acquire(wait:, **)
|
|
490
|
+
outcome(redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, limit, interval_ms, *wait]) })
|
|
491
|
+
end
|
|
492
|
+
end
|
|
493
|
+
|
|
494
|
+
# A token bucket holding +limit+ points, refilled continuously over
|
|
495
|
+
# +interval+. Each call spends any number of points; use #refund (or a
|
|
496
|
+
# negative #adjust) when the real cost turns out lower than estimated.
|
|
497
|
+
class Points < Base
|
|
498
|
+
# Fractions are written with %.14g, as Redis's Lua 5.1 does, so a
|
|
499
|
+
# server with a newer Lua (Dragonfly writes 2 as "2.0") stores the
|
|
500
|
+
# same text.
|
|
501
|
+
ACQUIRE = Limiter.acquiring(<<~LUA)
|
|
502
|
+
local now = tonumber(ARGV[1])
|
|
503
|
+
local cap = tonumber(ARGV[2])
|
|
504
|
+
local interval = tonumber(ARGV[3])
|
|
505
|
+
local cost = tonumber(ARGV[4])
|
|
506
|
+
local tokens = tonumber(redis.call('HGET', KEYS[1], 'tokens') or ARGV[2])
|
|
507
|
+
local last = tonumber(redis.call('HGET', KEYS[1], 'at') or ARGV[1])
|
|
508
|
+
tokens = math.min(cap, tokens + (now - last) * cap / interval)
|
|
509
|
+
if tokens >= cost then
|
|
510
|
+
redis.call('HSET', KEYS[1], 'tokens', string.format('%.14g', tokens - cost), 'at', now)
|
|
511
|
+
redis.call('PEXPIRE', KEYS[1], interval * 2)
|
|
512
|
+
redis.call('HSET', KEYS[2], 'last', now)
|
|
513
|
+
return 0
|
|
514
|
+
end
|
|
515
|
+
redis.call('HSET', KEYS[1], 'tokens', string.format('%.14g', tokens), 'at', now)
|
|
516
|
+
redis.call('PEXPIRE', KEYS[1], interval * 2)
|
|
517
|
+
return math.max(math.ceil((cost - tokens) * interval / cap), 1)
|
|
518
|
+
LUA
|
|
519
|
+
|
|
520
|
+
ADJUST = Redis::Script.new(<<~LUA)
|
|
521
|
+
local now = tonumber(ARGV[1])
|
|
522
|
+
local cap = tonumber(ARGV[2])
|
|
523
|
+
local interval = tonumber(ARGV[3])
|
|
524
|
+
local tokens = tonumber(redis.call('HGET', KEYS[1], 'tokens') or ARGV[2])
|
|
525
|
+
local last = tonumber(redis.call('HGET', KEYS[1], 'at') or ARGV[1])
|
|
526
|
+
tokens = math.min(cap, math.min(cap, tokens + (now - last) * cap / interval) - tonumber(ARGV[4]))
|
|
527
|
+
redis.call('HSET', KEYS[1], 'tokens', string.format('%.14g', tokens), 'at', now)
|
|
528
|
+
redis.call('PEXPIRE', KEYS[1], interval * 2)
|
|
529
|
+
return string.format('%.14g', tokens)
|
|
530
|
+
LUA
|
|
531
|
+
|
|
532
|
+
def initialize(name, limit, interval, **)
|
|
533
|
+
super
|
|
534
|
+
raise ArgumentError, "points limiters need an interval" unless interval_ms
|
|
535
|
+
end
|
|
536
|
+
|
|
537
|
+
def within_limit(points: 1, &)
|
|
538
|
+
raise ArgumentError, "points (#{points}) exceed the limiter's capacity (#{limit})" if points > limit
|
|
539
|
+
|
|
540
|
+
super
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
# Spends (positive) or returns (negative) points outside #within_limit.
|
|
544
|
+
# Returns the points now available.
|
|
545
|
+
def adjust(points)
|
|
546
|
+
Float(redis { |c| c.evalsha(ADJUST, [state_key], [now_ms, limit, interval_ms, points]) })
|
|
547
|
+
end
|
|
548
|
+
|
|
549
|
+
def refund(points) = adjust(-points)
|
|
550
|
+
|
|
551
|
+
# Points spent and not yet refilled; 0 when never used.
|
|
552
|
+
def used_command = ["HMGET", state_key, "tokens", "at"]
|
|
553
|
+
|
|
554
|
+
def used_from(reply)
|
|
555
|
+
tokens, at = reply
|
|
556
|
+
return 0 unless tokens
|
|
557
|
+
|
|
558
|
+
available = Float(tokens) + (now_ms - Integer(at)) * limit / interval_ms.to_f
|
|
559
|
+
(limit - available).ceil.clamp(0, limit)
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
private
|
|
563
|
+
|
|
564
|
+
def acquire(points:, wait:)
|
|
565
|
+
outcome(redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, limit, interval_ms, points, *wait]) })
|
|
566
|
+
end
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
# A limiter that never limits (and never touches Redis).
|
|
570
|
+
class Unlimited
|
|
571
|
+
attr_reader :name
|
|
572
|
+
|
|
573
|
+
def initialize(name) = @name = name.to_s
|
|
574
|
+
|
|
575
|
+
def type = "unlimited"
|
|
576
|
+
|
|
577
|
+
def within_limit(**) = yield(self)
|
|
578
|
+
|
|
579
|
+
def used = 0
|
|
580
|
+
|
|
581
|
+
# Like Points#adjust: as many points as anyone could want.
|
|
582
|
+
def adjust(_points) = Float::INFINITY
|
|
583
|
+
|
|
584
|
+
def refund(_points) = Float::INFINITY
|
|
585
|
+
|
|
586
|
+
def to_h = {"name" => name, "type" => type, "limit" => nil, "interval" => nil, "used" => 0}
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
# Reschedules jobs whose limiter is over its limit instead of failing
|
|
590
|
+
# them. After the limiter's +reschedule+ count (or
|
|
591
|
+
# config.max_rate_limit_retries) the OverLimit error is raised, so normal
|
|
592
|
+
# retry handling takes over.
|
|
593
|
+
class ServerMiddleware
|
|
594
|
+
include Middleware::ConfigAware
|
|
595
|
+
include Middleware::Stateless
|
|
596
|
+
|
|
597
|
+
def call(_job, payload, _queue)
|
|
598
|
+
yield
|
|
599
|
+
rescue OverLimit => e
|
|
600
|
+
count = payload["limited"].to_i + 1
|
|
601
|
+
raise if count > (e.limiter.reschedule || config.max_rate_limit_retries)
|
|
602
|
+
|
|
603
|
+
delay = custom_delay(e, count, payload) || default_delay(e, count, payload)
|
|
604
|
+
at = Robin.epoch_seconds(Robin.now_ms + Integer(delay * 1000))
|
|
605
|
+
stored = Robin.pristine(payload).except("at", "enqueued_at").merge({"limited" => count})
|
|
606
|
+
redis { |c| c.call("ZADD", config.key("schedule"), at, Robin.dump_json(stored)) }
|
|
607
|
+
Events.emit(config, "job.rate_limited", job: payload, limiter: e.limiter.name, delay: delay)
|
|
608
|
+
logger.info do
|
|
609
|
+
"#{payload.fetch("class")} jid=#{payload.fetch("jid")} over limit #{e.limiter.name.inspect}; rescheduled in #{delay.round(1)}s"
|
|
610
|
+
end
|
|
611
|
+
nil
|
|
612
|
+
end
|
|
613
|
+
|
|
614
|
+
private
|
|
615
|
+
|
|
616
|
+
# The limiter's backoff, else config.rate_limit_backoff: seconds, or
|
|
617
|
+
# nil for the default. One that raises or returns something else is
|
|
618
|
+
# reported, and the default used.
|
|
619
|
+
def custom_delay(error, count, payload)
|
|
620
|
+
backoff = error.limiter.backoff || config.rate_limit_backoff or return
|
|
621
|
+
seconds = backoff.call(count, error, payload)
|
|
622
|
+
return if seconds.nil?
|
|
623
|
+
return seconds if seconds.is_a?(Numeric) && seconds.real? && seconds.finite? && !seconds.negative?
|
|
624
|
+
|
|
625
|
+
raise ArgumentError, "rate limit backoff returned #{seconds.inspect}; expected seconds (>= 0) or nil"
|
|
626
|
+
rescue => e
|
|
627
|
+
config.handle_exception(e, {context: "rate limit backoff", job: payload})
|
|
628
|
+
nil
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
# Grows with each reschedule, at least as long as the limiter says
|
|
632
|
+
# capacity takes to free up, at most 5 minutes, with +/-15% jitter.
|
|
633
|
+
def default_delay(error, count, payload)
|
|
634
|
+
base = (payload["limiter_backoff"] || 5) * count
|
|
635
|
+
[base, error.retry_after.to_f].max.clamp(..300.0) * (0.85 + rand * 0.3)
|
|
636
|
+
end
|
|
637
|
+
end
|
|
638
|
+
end
|
|
639
|
+
end
|