robin 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/robin_resp/extconf.rb +29 -0
  13. data/ext/robin_resp/robin_resp.c +645 -0
  14. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  15. data/lib/generators/robin/install/install_generator.rb +39 -0
  16. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  17. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  18. data/lib/generators/robin/job/job_generator.rb +78 -0
  19. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  20. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  21. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  22. data/lib/puma/plugin/robin.rb +131 -0
  23. data/lib/robin/active_job.rb +144 -0
  24. data/lib/robin/api.rb +777 -0
  25. data/lib/robin/batch.rb +700 -0
  26. data/lib/robin/cancellation.rb +62 -0
  27. data/lib/robin/cli.rb +325 -0
  28. data/lib/robin/client.rb +355 -21
  29. data/lib/robin/config.rb +454 -4
  30. data/lib/robin/cron.rb +325 -0
  31. data/lib/robin/debounce.rb +108 -0
  32. data/lib/robin/deploy.rb +65 -0
  33. data/lib/robin/embedded.rb +53 -0
  34. data/lib/robin/encryption.rb +147 -0
  35. data/lib/robin/events.rb +60 -0
  36. data/lib/robin/expiry.rb +41 -0
  37. data/lib/robin/fetcher.rb +441 -0
  38. data/lib/robin/gvl.rb +62 -0
  39. data/lib/robin/health.rb +76 -0
  40. data/lib/robin/iterable_job.rb +294 -0
  41. data/lib/robin/job.rb +258 -0
  42. data/lib/robin/job_class.rb +157 -0
  43. data/lib/robin/job_logger.rb +121 -0
  44. data/lib/robin/launcher.rb +658 -0
  45. data/lib/robin/leader.rb +86 -0
  46. data/lib/robin/limiter.rb +639 -0
  47. data/lib/robin/manager.rb +125 -0
  48. data/lib/robin/metrics.rb +303 -0
  49. data/lib/robin/middleware.rb +165 -0
  50. data/lib/robin/opentelemetry.rb +88 -0
  51. data/lib/robin/periodic.rb +327 -0
  52. data/lib/robin/processor.rb +317 -0
  53. data/lib/robin/profiler.rb +140 -0
  54. data/lib/robin/progress.rb +46 -0
  55. data/lib/robin/rails/app_loader.rb +23 -0
  56. data/lib/robin/rails/current_attributes.rb +88 -0
  57. data/lib/robin/rails/error_reporter.rb +31 -0
  58. data/lib/robin/rails/notifications.rb +22 -0
  59. data/lib/robin/rails/railtie.rb +87 -0
  60. data/lib/robin/rails/transactional_push.rb +49 -0
  61. data/lib/robin/rails.rb +23 -0
  62. data/lib/robin/recovery.rb +135 -0
  63. data/lib/robin/redis/connection.rb +639 -0
  64. data/lib/robin/redis/pool.rb +190 -0
  65. data/lib/robin/redis/resp.rb +77 -0
  66. data/lib/robin/reliable_push.rb +153 -0
  67. data/lib/robin/retrier.rb +284 -0
  68. data/lib/robin/scheduler.rb +102 -0
  69. data/lib/robin/scripts.rb +331 -0
  70. data/lib/robin/statsd.rb +135 -0
  71. data/lib/robin/swarm.rb +319 -0
  72. data/lib/robin/systemd.rb +47 -0
  73. data/lib/robin/testing.rb +158 -0
  74. data/lib/robin/top.rb +178 -0
  75. data/lib/robin/unique.rb +198 -0
  76. data/lib/robin/version.rb +5 -0
  77. data/lib/robin/warmup.rb +94 -0
  78. data/lib/robin/web/api.rb +580 -0
  79. data/lib/robin/web/batch_status.rb +44 -0
  80. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  81. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  82. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  83. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  84. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  86. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  87. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  88. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  89. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  90. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  91. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  92. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  93. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  94. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  95. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  96. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  97. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  98. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  99. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  100. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  101. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  102. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  103. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  104. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  105. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  106. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  107. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  108. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  109. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  110. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  111. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  112. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  113. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  114. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  115. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  116. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  117. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  118. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  119. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  120. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  121. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  122. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  123. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  124. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  125. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  126. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  127. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  128. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  129. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  130. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  131. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  132. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  133. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  134. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  135. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  136. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  137. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  138. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  139. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  140. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  141. data/lib/robin/web/dist/index.html +19 -0
  142. data/lib/robin/web/extension.rb +193 -0
  143. data/lib/robin/web/permissions.rb +40 -0
  144. data/lib/robin/web/prometheus.rb +79 -0
  145. data/lib/robin/web/router.rb +150 -0
  146. data/lib/robin/web/static.rb +110 -0
  147. data/lib/robin/web.rb +196 -0
  148. data/lib/robin.rb +258 -8
  149. data/sig/manifest.yaml +9 -0
  150. data/sig/robin/api.rbs +442 -0
  151. data/sig/robin/batch.rbs +292 -0
  152. data/sig/robin/cancellation.rbs +25 -0
  153. data/sig/robin/cli.rbs +102 -0
  154. data/sig/robin/client.rbs +118 -0
  155. data/sig/robin/config.rbs +341 -0
  156. data/sig/robin/cron.rbs +149 -0
  157. data/sig/robin/debounce.rbs +36 -0
  158. data/sig/robin/deploy.rbs +24 -0
  159. data/sig/robin/embedded.rbs +25 -0
  160. data/sig/robin/encryption.rbs +49 -0
  161. data/sig/robin/events.rbs +10 -0
  162. data/sig/robin/expiry.rbs +17 -0
  163. data/sig/robin/fetcher.rbs +134 -0
  164. data/sig/robin/gvl.rbs +39 -0
  165. data/sig/robin/health.rbs +30 -0
  166. data/sig/robin/iterable_job.rbs +99 -0
  167. data/sig/robin/job.rbs +173 -0
  168. data/sig/robin/job_class.rbs +55 -0
  169. data/sig/robin/job_logger.rbs +56 -0
  170. data/sig/robin/launcher.rbs +191 -0
  171. data/sig/robin/leader.rbs +43 -0
  172. data/sig/robin/limiter.rbs +317 -0
  173. data/sig/robin/manager.rbs +54 -0
  174. data/sig/robin/metrics.rbs +122 -0
  175. data/sig/robin/middleware.rbs +106 -0
  176. data/sig/robin/opentelemetry.rbs +51 -0
  177. data/sig/robin/periodic.rbs +81 -0
  178. data/sig/robin/processor.rbs +107 -0
  179. data/sig/robin/profiler.rbs +60 -0
  180. data/sig/robin/progress.rbs +22 -0
  181. data/sig/robin/rails/app_loader.rbs +10 -0
  182. data/sig/robin/rails/current_attributes.rbs +44 -0
  183. data/sig/robin/rails/error_reporter.rbs +21 -0
  184. data/sig/robin/rails/notifications.rbs +9 -0
  185. data/sig/robin/rails/transactional_push.rbs +16 -0
  186. data/sig/robin/rails.rbs +8 -0
  187. data/sig/robin/recovery.rbs +29 -0
  188. data/sig/robin/redis/connection.rbs +224 -0
  189. data/sig/robin/redis/pool.rbs +89 -0
  190. data/sig/robin/redis/resp.rbs +29 -0
  191. data/sig/robin/reliable_push.rbs +77 -0
  192. data/sig/robin/retrier.rbs +72 -0
  193. data/sig/robin/scheduler.rbs +40 -0
  194. data/sig/robin/scripts.rbs +36 -0
  195. data/sig/robin/statsd.rbs +55 -0
  196. data/sig/robin/swarm.rbs +133 -0
  197. data/sig/robin/systemd.rbs +13 -0
  198. data/sig/robin/testing.rbs +80 -0
  199. data/sig/robin/top.rbs +79 -0
  200. data/sig/robin/unique.rbs +72 -0
  201. data/sig/robin/version.rbs +3 -0
  202. data/sig/robin/warmup.rbs +44 -0
  203. data/sig/robin/web/api.rbs +196 -0
  204. data/sig/robin/web/batch_status.rbs +25 -0
  205. data/sig/robin/web/extension.rbs +106 -0
  206. data/sig/robin/web/permissions.rbs +14 -0
  207. data/sig/robin/web/prometheus.rbs +33 -0
  208. data/sig/robin/web/router.rbs +53 -0
  209. data/sig/robin/web/static.rbs +42 -0
  210. data/sig/robin/web.rbs +140 -0
  211. data/sig/robin.rbs +116 -0
  212. data/sig/robin_resp.rbs +55 -0
  213. metadata +274 -44
  214. data/bin/robin +0 -5
  215. data/lib/robin/actions/authenticate.rb +0 -38
  216. data/lib/robin/actions/empty_tweet.rb +0 -7
  217. data/lib/robin/actions/followers.rb +0 -22
  218. data/lib/robin/actions/help.rb +0 -16
  219. data/lib/robin/actions/home_timeline.rb +0 -9
  220. data/lib/robin/actions/new_tweet.rb +0 -11
  221. data/lib/robin/actions/unknown_flag.rb +0 -11
  222. data/lib/robin/actions/user_timeline.rb +0 -19
  223. data/lib/robin/args_parser.rb +0 -70
  224. data/lib/robin/pin_authenticator.rb +0 -31
  225. data/lib/robin/user.rb +0 -36
@@ -0,0 +1,639 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Distributed rate limiting, usable from jobs or any other Ruby code.
5
+ #
6
+ # STRIPE = Robin::Limiter.concurrent("stripe", 10) # at most 10 at once
7
+ # GITHUB = Robin::Limiter.bucket("github", 5_000, :hour) # 5000 per calendar hour
8
+ # SMS = Robin::Limiter.window("sms", 10, :minute) # 10 in any 60s window
9
+ # EMAIL = Robin::Limiter.leaky("email", 60, 60) # drains 1/s, bursts to 60
10
+ # OPENAI = Robin::Limiter.points("openai", 90_000, 60) # token budget per minute
11
+ #
12
+ # STRIPE.within_limit { charge! }
13
+ # OPENAI.within_limit(points: tokens) { complete! }
14
+ # GITHUB.within_limit(used: 3) { sync_three_repos! } # bucket and window
15
+ #
16
+ # Limiters keep their state in the configured Redis; Limiter.via sends a
17
+ # thread's limiter calls to another one (e.g. a shard of their own):
18
+ #
19
+ # Robin::Limiter.via(LIMITER_REDIS) { STRIPE.within_limit { charge! } }
20
+ #
21
+ # The web UI lists the limiters in the configured Redis only.
22
+ #
23
+ # #within_limit waits up to +wait_timeout+ seconds for capacity and
24
+ # otherwise raises Robin::Limiter::OverLimit. Inside a job, Robin catches
25
+ # OverLimit and reschedules the job with a backoff; that doesn't count as
26
+ # a failure or use up retries.
27
+ #
28
+ # Every kind of limiter also takes:
29
+ #
30
+ # reschedule:: how many times a job that hits this limiter is rescheduled
31
+ # before the OverLimit counts as a failure (default
32
+ # config.max_rate_limit_retries; 0 fails at once)
33
+ # backoff:: a callable (count, error, payload) returning the seconds to
34
+ # wait before running the job again, or nil for the default
35
+ # (see also config.rate_limit_backoff)
36
+ # policy:: what #within_limit does when it can't get capacity in
37
+ # time: :raise (the default) raises OverLimit; :skip
38
+ # returns nil without running the block, for work that
39
+ # can be dropped (e.g. refreshing a cache)
40
+ #
41
+ # Robin::Limiter.unlimited never limits: swap it in where a limit
42
+ # shouldn't apply, e.g. in tests.
43
+ module Limiter
44
+ class OverLimit < Robin::Error
45
+ attr_reader :limiter, :retry_after
46
+
47
+ def initialize(limiter, retry_after = nil)
48
+ @limiter = limiter
49
+ @retry_after = retry_after
50
+ super("rate limiter #{limiter.name.inspect} (#{limiter.type}) is over its limit")
51
+ end
52
+ end
53
+
54
+ # The end of every acquiring script (see Limiter.acquiring).
55
+ COUNT_OVER = <<~LUA
56
+ if taken == false or (type(taken) == 'number' and taken > 0) then
57
+ if (taken or tonumber(ARGV[#ARGV - 1])) > tonumber(ARGV[#ARGV]) then
58
+ redis.call('HINCRBY', KEYS[2], 'over', 1)
59
+ end
60
+ end
61
+ return taken
62
+ LUA
63
+
64
+ # A limiter's acquiring script, from +body+: Lua that takes capacity
65
+ # (KEYS[1] its state, KEYS[2] its definition) and returns 0, or a
66
+ # token, when it took some, else how many ms until some frees up, or
67
+ # false when it can't tell. The script also counts the limiter over
68
+ # its limit when the wait is longer than the caller will wait. Its
69
+ # last two ARGV, after the body's own: the wait (ms) to assume when
70
+ # the body can't tell, and how long (ms) the caller will wait.
71
+ def self.acquiring(body) = Redis::Script.new("local taken = (function()\n#{body}\nend)()\n#{COUNT_OVER}")
72
+
73
+ INTERVALS = {second: 1, minute: 60, hour: 3_600, day: 86_400}.freeze
74
+ TYPES = {"concurrent" => "Concurrent", "bucket" => "Bucket", "window" => "Window",
75
+ "leaky" => "Leaky", "points" => "Points"}.freeze
76
+ META_TTL = 30 * 86_400
77
+ REGISTER_EVERY = 600
78
+
79
+ # The definition each limiter name last wrote to each Redis, and when:
80
+ # {[pool id, meta key] => [[type, limit, interval], monotonic time]}.
81
+ # Shared by every limiter object, so one built inside perform doesn't
82
+ # write its definition on every call.
83
+ @registered = {}
84
+
85
+ class << self
86
+ attr_reader :registered # :nodoc:
87
+
88
+ # Sends every limiter call this thread makes inside the block to
89
+ # +pool+ (a Robin::Redis::Pool) instead of the configured Redis.
90
+ def via(pool)
91
+ previous = Thread.current[:robin_limiter_pool]
92
+ Thread.current[:robin_limiter_pool] = pool
93
+ yield
94
+ ensure
95
+ Thread.current[:robin_limiter_pool] = previous
96
+ end
97
+
98
+ # The pool set by an enclosing Limiter.via block, if any.
99
+ def current_pool = Thread.current[:robin_limiter_pool]
100
+
101
+ def concurrent(name, size, wait_timeout: 5, lock_timeout: 30, **)
102
+ Concurrent.new(name, size, wait_timeout: wait_timeout, lock_timeout: lock_timeout, **)
103
+ end
104
+
105
+ def bucket(name, count, interval, **) = Bucket.new(name, count, interval, **)
106
+
107
+ def window(name, count, interval, **) = Window.new(name, count, interval, **)
108
+
109
+ def leaky(name, size, interval, **) = Leaky.new(name, size, interval, **)
110
+
111
+ def points(name, points, interval, **) = Points.new(name, points, interval, **)
112
+
113
+ def unlimited(name = "unlimited") = Unlimited.new(name)
114
+
115
+ # Seconds for a :second/:minute/:hour/:day symbol (or string), a
116
+ # number of seconds, or nil.
117
+ def seconds(interval)
118
+ case interval
119
+ when Symbol, String then INTERVALS.fetch(interval.to_sym) { raise ArgumentError, "unknown interval #{interval.inspect}" }
120
+ when nil then nil
121
+ else Float(interval)
122
+ end
123
+ end
124
+
125
+ # Every limiter used in the last 30 days, with current usage, in
126
+ # two round trips.
127
+ def list(config: Robin.config)
128
+ metas = Scripts.member_hashes(config, config.key("limiters"), "#{config.key("lmtr")}:")
129
+ expired, live = metas.partition { |_name, meta| meta.empty? }
130
+ limiters = live.map { |name, meta| from_meta(name, meta, config) }
131
+ usage = config.redis do |c|
132
+ c.pipelined do |p|
133
+ limiters.compact.each { |limiter| p.call(*limiter.used_command) }
134
+ p.call("SREM", config.key("limiters"), *expired.map(&:first)) unless expired.empty?
135
+ end
136
+ end
137
+ live.zip(limiters).map do |(name, meta), limiter|
138
+ {
139
+ "name" => name, "type" => meta.fetch("type"), "limit" => Integer(meta.fetch("limit")),
140
+ "interval" => interval_from(meta), "used" => limiter ? limiter.used_from(usage.shift) : 0,
141
+ "over_limit_count" => meta["over"].to_i, "last_used_at" => meta["last"]&.to_i
142
+ }
143
+ end
144
+ end
145
+
146
+ # Clears a limiter's state and over-limit counter, in two round trips.
147
+ def reset!(name, config: Robin.config)
148
+ clear(name, config) { |p, known| p.call("HSET", config.key("lmtr", name), "over", 0) if known }
149
+ true
150
+ end
151
+
152
+ # Forgets a limiter: its state and its definition, in two round trips.
153
+ def delete!(name, config: Robin.config)
154
+ clear(name, config) do |p|
155
+ p.call("DEL", config.key("lmtr", name))
156
+ p.call("SREM", config.key("limiters"), name)
157
+ end
158
+ @registered.delete([config.redis_pool.object_id, config.key("lmtr", name)])
159
+ true
160
+ end
161
+
162
+ private
163
+
164
+ # Clears limiter +name+'s state, along with what the block queues
165
+ # on the same pipeline; it's told whether the limiter has a
166
+ # definition (one never used, or expired, has no state to clear).
167
+ def clear(name, config)
168
+ meta = config.redis { |c| c.call("HGETALL", config.key("lmtr", name)) }.each_slice(2).to_h
169
+ limiter = from_meta(name, meta, config) unless meta.empty?
170
+ config.redis do |c|
171
+ c.pipelined do |p|
172
+ p.call(*limiter.reset_command) if limiter
173
+ yield p, !meta.empty?
174
+ end
175
+ end
176
+ end
177
+
178
+ def from_meta(name, meta, config)
179
+ klass = TYPES[meta.fetch("type")]
180
+ klass && const_get(klass).new(name, meta.fetch("limit"), interval_from(meta), config: config)
181
+ end
182
+
183
+ def interval_from(meta)
184
+ raw = meta.fetch("interval")
185
+ Float(raw) unless raw.empty?
186
+ end
187
+ end
188
+
189
+ class Base
190
+ POLICIES = %i[raise skip].freeze
191
+
192
+ attr_reader :name, :limit, :interval, :interval_ms, :wait_timeout, :reschedule, :backoff, :policy
193
+
194
+ def initialize(name, limit, interval, wait_timeout: 5, reschedule: nil, backoff: nil, policy: :raise, config: nil)
195
+ @name = name.to_s
196
+ raise ArgumentError, "limiter name can't be blank" if @name.empty?
197
+
198
+ @limit = Integer(limit)
199
+ raise ArgumentError, "limit must be positive" unless @limit.positive?
200
+
201
+ @interval = seconds = Limiter.seconds(interval)
202
+ @interval_ms = Integer(seconds * 1000) if seconds
203
+ @wait_timeout = Float(wait_timeout)
204
+ @reschedule = reschedule && Integer(reschedule)
205
+ raise ArgumentError, "reschedule can't be negative" if @reschedule&.negative?
206
+ raise ArgumentError, "backoff must respond to #call" unless backoff.nil? || backoff.respond_to?(:call)
207
+
208
+ raise ArgumentError, "policy must be :raise or :skip, not #{policy.inspect}" unless POLICIES.include?(policy)
209
+
210
+ @backoff = backoff
211
+ @policy = policy
212
+ @config = config
213
+ end
214
+
215
+ def config = @config || Robin.config
216
+
217
+ def type = self.class.name.split("::").last.downcase
218
+
219
+ # Runs the block once capacity is available. If it doesn't become
220
+ # available within +wait_timeout+, raises OverLimit or (with policy:
221
+ # :skip) returns nil without running the block.
222
+ def within_limit(**opts)
223
+ register
224
+ token = wait_for_capacity(opts) or return
225
+ begin
226
+ yield self
227
+ ensure
228
+ release(token)
229
+ end
230
+ end
231
+
232
+ def over_limit!
233
+ redis { |c| c.call("HINCRBY", meta_key, "over", 1) }
234
+ rescue
235
+ nil
236
+ end
237
+
238
+ def reset_state
239
+ redis { |c| c.call(*reset_command) }
240
+ end
241
+
242
+ # The command #reset_state sends.
243
+ def reset_command = ["DEL", state_key]
244
+
245
+ # How many uses are counted now: what the reply to each kind's
246
+ # #used_command (Limiter.list sends them together) means.
247
+ def used = used_from(redis { |c| c.call(*used_command) })
248
+
249
+ def used_from(reply) = reply
250
+
251
+ def to_h = {"name" => name, "type" => type, "limit" => limit, "interval" => interval, "used" => used}
252
+
253
+ private
254
+
255
+ def meta_key = config.key("lmtr", name)
256
+
257
+ def state_key = config.key("lmtr", name, "s")
258
+
259
+ def now_ms = Robin.now_ms
260
+
261
+ def release(_token)
262
+ end
263
+
264
+ # Scripts return 0 when they took capacity, else how many ms until
265
+ # some frees up. Returns [token, wait_ms].
266
+ def outcome(wait) = [wait.zero?, wait]
267
+
268
+ # Retries #acquire until it succeeds or waiting longer would pass the
269
+ # deadline. Limiters that know when capacity frees up say so (in ms);
270
+ # the others are polled with a short exponential backoff. Returns
271
+ # the token, or nil when skipping.
272
+ #
273
+ # The script counts the limiter over its limit when it gives up, in
274
+ # the same round trip: it's told the wait to assume (the backoff)
275
+ # and how long is left (+wait+, in ms).
276
+ def wait_for_capacity(opts)
277
+ deadline = monotonic + @wait_timeout
278
+ attempt = 0
279
+ loop do
280
+ guess = [0.01 * (2**attempt), 0.25].min * 1000
281
+ left = (deadline - monotonic) * 1000
282
+ token, wait_ms = acquire(**opts, wait: [guess, left])
283
+ return token if token
284
+
285
+ wait = wait_ms || guess
286
+ if wait > left
287
+ return if policy == :skip
288
+
289
+ raise OverLimit.new(self, wait_ms && wait_ms / 1000.0)
290
+ end
291
+ attempt += 1
292
+ sleep(wait / 1000.0 + rand * 0.005)
293
+ end
294
+ end
295
+
296
+ # Writes the definition for the web UI, once per process and Redis
297
+ # every 10 minutes, or as soon as a limiter of the same name with a
298
+ # different definition writes its own.
299
+ def register
300
+ target = pool
301
+ cache_key = [target.object_id, meta_key] #: [Integer, String]
302
+ definition = [type, limit, interval] #: [String, Integer, Integer | Float | nil]
303
+ written, at = Limiter.registered[cache_key]
304
+ return if written == definition && monotonic - at < REGISTER_EVERY
305
+
306
+ target.with do |c|
307
+ c.pipelined do |p|
308
+ p.call("HSET", meta_key, "type", type, "limit", limit, "interval", interval)
309
+ p.call("EXPIRE", meta_key, META_TTL)
310
+ p.call("SADD", config.key("limiters"), name)
311
+ end
312
+ end
313
+ Limiter.registered[cache_key] = [definition, monotonic]
314
+ end
315
+
316
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
317
+
318
+ def pool = Limiter.current_pool || config.redis_pool
319
+
320
+ def redis(&) = pool.with(&)
321
+ end
322
+
323
+ # Lets one call count as several uses (+used:+), e.g. an API call that
324
+ # covers a batch of records.
325
+ module Uses
326
+ def within_limit(used: 1, &)
327
+ unless used.instance_of?(Integer) && used.between?(1, limit)
328
+ raise ArgumentError, "used (#{used.inspect}) must be a whole number from 1 to the limit (#{limit})"
329
+ end
330
+
331
+ super
332
+ end
333
+ end
334
+
335
+ # At most +size+ holders at once. Holders lease their slot for
336
+ # +lock_timeout+ seconds so a crashed process can't leak capacity.
337
+ class Concurrent < Base
338
+ # Returns the lease token, or false when full.
339
+ ACQUIRE = Limiter.acquiring(<<~LUA)
340
+ local now = tonumber(ARGV[1])
341
+ redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now)
342
+ if redis.call('ZCARD', KEYS[1]) < tonumber(ARGV[2]) then
343
+ redis.call('ZADD', KEYS[1], now + tonumber(ARGV[3]), ARGV[4])
344
+ redis.call('PEXPIRE', KEYS[1], tonumber(ARGV[3]) + 60000)
345
+ redis.call('HSET', KEYS[2], 'last', now)
346
+ return ARGV[4]
347
+ end
348
+ return false
349
+ LUA
350
+
351
+ attr_reader :lock_timeout, :lock_timeout_ms
352
+
353
+ def initialize(name, size, _interval = nil, lock_timeout: 30, **)
354
+ super(name, size, nil, **)
355
+ @lock_timeout = Float(lock_timeout)
356
+ @lock_timeout_ms = Integer(@lock_timeout * 1000)
357
+ end
358
+
359
+ def used_command = ["ZCOUNT", state_key, "(#{now_ms}", "+inf"]
360
+
361
+ private
362
+
363
+ # The lease token, or nil when full (no wait is announced: holders
364
+ # can release at any time, so callers poll).
365
+ def acquire(wait:, **)
366
+ redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, limit, lock_timeout_ms, Robin.generate_jid, *wait]) }
367
+ end
368
+
369
+ def release(token)
370
+ redis { |c| c.call("ZREM", state_key, token) }
371
+ end
372
+ end
373
+
374
+ # +limit+ uses per calendar interval (each second/minute/hour/day, or
375
+ # any number of seconds, aligned to the epoch).
376
+ class Bucket < Base
377
+ include Uses
378
+
379
+ # ARGV: limit, ttl (ms), now (ms), uses
380
+ ACQUIRE = Limiter.acquiring(<<~LUA)
381
+ local used = tonumber(ARGV[4])
382
+ local n = redis.call('INCRBY', KEYS[1], used)
383
+ if n == used then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
384
+ if n > tonumber(ARGV[1]) then
385
+ redis.call('DECRBY', KEYS[1], used)
386
+ return math.max(redis.call('PTTL', KEYS[1]), 1)
387
+ end
388
+ redis.call('HSET', KEYS[2], 'last', ARGV[3])
389
+ return 0
390
+ LUA
391
+
392
+ def initialize(name, limit, interval, **)
393
+ super
394
+ raise ArgumentError, "bucket limiters need an interval" unless interval_ms
395
+ end
396
+
397
+ def used_command = ["GET", bucket_key(now_ms)]
398
+
399
+ def used_from(reply) = reply.to_i
400
+
401
+ def reset_command = ["DEL", bucket_key(now_ms)]
402
+
403
+ private
404
+
405
+ def bucket_key(now) = config.key("lmtr", name, "b", now / interval_ms)
406
+
407
+ def acquire(used:, wait:)
408
+ now = now_ms
409
+ ttl = interval_ms - (now % interval_ms)
410
+ outcome(redis { |c| c.evalsha(ACQUIRE, [bucket_key(now), meta_key], [limit, ttl, now, used, *wait]) })
411
+ end
412
+ end
413
+
414
+ # At most +limit+ uses in any rolling +interval+.
415
+ class Window < Base
416
+ include Uses
417
+
418
+ # ARGV: now (ms), interval (ms), limit, unique id, uses. Each use is
419
+ # a member; when they don't fit, the wait is until enough expire.
420
+ ACQUIRE = Limiter.acquiring(<<~LUA)
421
+ local now = tonumber(ARGV[1])
422
+ local interval = tonumber(ARGV[2])
423
+ local used = tonumber(ARGV[5])
424
+ redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now - interval)
425
+ local over = redis.call('ZCARD', KEYS[1]) + used - tonumber(ARGV[3])
426
+ if over <= 0 then
427
+ for i = 1, used do redis.call('ZADD', KEYS[1], now, ARGV[4] .. ':' .. i) end
428
+ redis.call('PEXPIRE', KEYS[1], interval + 1000)
429
+ redis.call('HSET', KEYS[2], 'last', now)
430
+ return 0
431
+ end
432
+ local freeing = redis.call('ZRANGE', KEYS[1], over - 1, over - 1, 'WITHSCORES')
433
+ return math.max(tonumber(freeing[2]) + interval - now, 1)
434
+ LUA
435
+
436
+ def initialize(name, limit, interval, **)
437
+ super
438
+ raise ArgumentError, "window limiters need an interval" unless interval_ms
439
+ end
440
+
441
+ def used_command = ["ZCOUNT", state_key, "(#{now_ms - interval_ms}", "+inf"]
442
+
443
+ private
444
+
445
+ def acquire(used:, wait:)
446
+ outcome(redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, interval_ms, limit, Robin.generate_jid, used, *wait]) })
447
+ end
448
+ end
449
+
450
+ # A bucket of capacity +limit+ that drains at limit/interval per second:
451
+ # allows bursts up to +limit+, then a steady rate.
452
+ class Leaky < Base
453
+ # Fractions are written with %.14g, as Redis's Lua 5.1 does, so a
454
+ # server with a newer Lua (Dragonfly writes 2 as "2.0") stores the
455
+ # same text.
456
+ ACQUIRE = Limiter.acquiring(<<~LUA)
457
+ local now = tonumber(ARGV[1])
458
+ local size = tonumber(ARGV[2])
459
+ local interval = tonumber(ARGV[3])
460
+ local level = tonumber(redis.call('HGET', KEYS[1], 'level') or '0')
461
+ local last = tonumber(redis.call('HGET', KEYS[1], 'at') or ARGV[1])
462
+ level = math.max(0, level - (now - last) * size / interval)
463
+ if level + 1 <= size then
464
+ redis.call('HSET', KEYS[1], 'level', string.format('%.14g', level + 1), 'at', now)
465
+ redis.call('PEXPIRE', KEYS[1], interval + 1000)
466
+ redis.call('HSET', KEYS[2], 'last', now)
467
+ return 0
468
+ end
469
+ redis.call('HSET', KEYS[1], 'level', string.format('%.14g', level), 'at', now)
470
+ return math.max(math.ceil((level + 1 - size) * interval / size), 1)
471
+ LUA
472
+
473
+ def initialize(name, limit, interval, **)
474
+ super
475
+ raise ArgumentError, "leaky limiters need an interval" unless interval_ms
476
+ end
477
+
478
+ # Current level; 0 when the limiter has never been used.
479
+ def used_command = ["HMGET", state_key, "level", "at"]
480
+
481
+ def used_from(reply)
482
+ level, at = reply
483
+ drained = (now_ms - at.to_i) * limit / interval_ms.to_f
484
+ [level.to_f - drained, 0].max.ceil
485
+ end
486
+
487
+ private
488
+
489
+ def acquire(wait:, **)
490
+ outcome(redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, limit, interval_ms, *wait]) })
491
+ end
492
+ end
493
+
494
+ # A token bucket holding +limit+ points, refilled continuously over
495
+ # +interval+. Each call spends any number of points; use #refund (or a
496
+ # negative #adjust) when the real cost turns out lower than estimated.
497
+ class Points < Base
498
+ # Fractions are written with %.14g, as Redis's Lua 5.1 does, so a
499
+ # server with a newer Lua (Dragonfly writes 2 as "2.0") stores the
500
+ # same text.
501
+ ACQUIRE = Limiter.acquiring(<<~LUA)
502
+ local now = tonumber(ARGV[1])
503
+ local cap = tonumber(ARGV[2])
504
+ local interval = tonumber(ARGV[3])
505
+ local cost = tonumber(ARGV[4])
506
+ local tokens = tonumber(redis.call('HGET', KEYS[1], 'tokens') or ARGV[2])
507
+ local last = tonumber(redis.call('HGET', KEYS[1], 'at') or ARGV[1])
508
+ tokens = math.min(cap, tokens + (now - last) * cap / interval)
509
+ if tokens >= cost then
510
+ redis.call('HSET', KEYS[1], 'tokens', string.format('%.14g', tokens - cost), 'at', now)
511
+ redis.call('PEXPIRE', KEYS[1], interval * 2)
512
+ redis.call('HSET', KEYS[2], 'last', now)
513
+ return 0
514
+ end
515
+ redis.call('HSET', KEYS[1], 'tokens', string.format('%.14g', tokens), 'at', now)
516
+ redis.call('PEXPIRE', KEYS[1], interval * 2)
517
+ return math.max(math.ceil((cost - tokens) * interval / cap), 1)
518
+ LUA
519
+
520
+ ADJUST = Redis::Script.new(<<~LUA)
521
+ local now = tonumber(ARGV[1])
522
+ local cap = tonumber(ARGV[2])
523
+ local interval = tonumber(ARGV[3])
524
+ local tokens = tonumber(redis.call('HGET', KEYS[1], 'tokens') or ARGV[2])
525
+ local last = tonumber(redis.call('HGET', KEYS[1], 'at') or ARGV[1])
526
+ tokens = math.min(cap, math.min(cap, tokens + (now - last) * cap / interval) - tonumber(ARGV[4]))
527
+ redis.call('HSET', KEYS[1], 'tokens', string.format('%.14g', tokens), 'at', now)
528
+ redis.call('PEXPIRE', KEYS[1], interval * 2)
529
+ return string.format('%.14g', tokens)
530
+ LUA
531
+
532
+ def initialize(name, limit, interval, **)
533
+ super
534
+ raise ArgumentError, "points limiters need an interval" unless interval_ms
535
+ end
536
+
537
+ def within_limit(points: 1, &)
538
+ raise ArgumentError, "points (#{points}) exceed the limiter's capacity (#{limit})" if points > limit
539
+
540
+ super
541
+ end
542
+
543
+ # Spends (positive) or returns (negative) points outside #within_limit.
544
+ # Returns the points now available.
545
+ def adjust(points)
546
+ Float(redis { |c| c.evalsha(ADJUST, [state_key], [now_ms, limit, interval_ms, points]) })
547
+ end
548
+
549
+ def refund(points) = adjust(-points)
550
+
551
+ # Points spent and not yet refilled; 0 when never used.
552
+ def used_command = ["HMGET", state_key, "tokens", "at"]
553
+
554
+ def used_from(reply)
555
+ tokens, at = reply
556
+ return 0 unless tokens
557
+
558
+ available = Float(tokens) + (now_ms - Integer(at)) * limit / interval_ms.to_f
559
+ (limit - available).ceil.clamp(0, limit)
560
+ end
561
+
562
+ private
563
+
564
+ def acquire(points:, wait:)
565
+ outcome(redis { |c| c.evalsha(ACQUIRE, [state_key, meta_key], [now_ms, limit, interval_ms, points, *wait]) })
566
+ end
567
+ end
568
+
569
+ # A limiter that never limits (and never touches Redis).
570
+ class Unlimited
571
+ attr_reader :name
572
+
573
+ def initialize(name) = @name = name.to_s
574
+
575
+ def type = "unlimited"
576
+
577
+ def within_limit(**) = yield(self)
578
+
579
+ def used = 0
580
+
581
+ # Like Points#adjust: as many points as anyone could want.
582
+ def adjust(_points) = Float::INFINITY
583
+
584
+ def refund(_points) = Float::INFINITY
585
+
586
+ def to_h = {"name" => name, "type" => type, "limit" => nil, "interval" => nil, "used" => 0}
587
+ end
588
+
589
+ # Reschedules jobs whose limiter is over its limit instead of failing
590
+ # them. After the limiter's +reschedule+ count (or
591
+ # config.max_rate_limit_retries) the OverLimit error is raised, so normal
592
+ # retry handling takes over.
593
+ class ServerMiddleware
594
+ include Middleware::ConfigAware
595
+ include Middleware::Stateless
596
+
597
+ def call(_job, payload, _queue)
598
+ yield
599
+ rescue OverLimit => e
600
+ count = payload["limited"].to_i + 1
601
+ raise if count > (e.limiter.reschedule || config.max_rate_limit_retries)
602
+
603
+ delay = custom_delay(e, count, payload) || default_delay(e, count, payload)
604
+ at = Robin.epoch_seconds(Robin.now_ms + Integer(delay * 1000))
605
+ stored = Robin.pristine(payload).except("at", "enqueued_at").merge({"limited" => count})
606
+ redis { |c| c.call("ZADD", config.key("schedule"), at, Robin.dump_json(stored)) }
607
+ Events.emit(config, "job.rate_limited", job: payload, limiter: e.limiter.name, delay: delay)
608
+ logger.info do
609
+ "#{payload.fetch("class")} jid=#{payload.fetch("jid")} over limit #{e.limiter.name.inspect}; rescheduled in #{delay.round(1)}s"
610
+ end
611
+ nil
612
+ end
613
+
614
+ private
615
+
616
+ # The limiter's backoff, else config.rate_limit_backoff: seconds, or
617
+ # nil for the default. One that raises or returns something else is
618
+ # reported, and the default used.
619
+ def custom_delay(error, count, payload)
620
+ backoff = error.limiter.backoff || config.rate_limit_backoff or return
621
+ seconds = backoff.call(count, error, payload)
622
+ return if seconds.nil?
623
+ return seconds if seconds.is_a?(Numeric) && seconds.real? && seconds.finite? && !seconds.negative?
624
+
625
+ raise ArgumentError, "rate limit backoff returned #{seconds.inspect}; expected seconds (>= 0) or nil"
626
+ rescue => e
627
+ config.handle_exception(e, {context: "rate limit backoff", job: payload})
628
+ nil
629
+ end
630
+
631
+ # Grows with each reschedule, at least as long as the limiter says
632
+ # capacity takes to free up, at most 5 minutes, with +/-15% jitter.
633
+ def default_delay(error, count, payload)
634
+ base = (payload["limiter_backoff"] || 5) * count
635
+ [base, error.retry_after.to_f].max.clamp(..300.0) * (0.85 + rand * 0.3)
636
+ end
637
+ end
638
+ end
639
+ end