robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Runs a Robin server: worker threads for every capsule, the scheduler,
|
|
3
|
+
# and a heartbeat thread that publishes process state, flushes metrics,
|
|
4
|
+
# takes part in leader election and reacts to remote control signals.
|
|
5
|
+
class Launcher
|
|
6
|
+
# A housekeeping hook (see #every_beat), given the heartbeat's
|
|
7
|
+
# connection.
|
|
8
|
+
type hook = ^(Redis::Connection conn) -> void
|
|
9
|
+
|
|
10
|
+
STOPPING_MARGIN: Integer
|
|
11
|
+
|
|
12
|
+
HARD_SHUTDOWN_GRACE: Integer
|
|
13
|
+
|
|
14
|
+
SLOW_REDIS_US: Integer
|
|
15
|
+
|
|
16
|
+
IDLE_GC_AFTER: Integer
|
|
17
|
+
|
|
18
|
+
GVL_HIGH: Integer
|
|
19
|
+
|
|
20
|
+
GVL_HIGH_BEATS: Integer
|
|
21
|
+
|
|
22
|
+
MINIMUM_REDIS: Array[Integer]
|
|
23
|
+
|
|
24
|
+
@embedded: bool
|
|
25
|
+
@scheduler: Scheduler
|
|
26
|
+
@quiet: bool
|
|
27
|
+
@done: bool
|
|
28
|
+
@last: Hash[Symbol, Float]
|
|
29
|
+
@acks: Thread::Queue[Fetcher?]
|
|
30
|
+
@hooks: Hash[Symbol, Array[hook]]
|
|
31
|
+
@suspects: Array[[String, String?]]
|
|
32
|
+
@gvl: GVL
|
|
33
|
+
@gvl_high: Integer
|
|
34
|
+
@wake_mutex: Thread::Mutex
|
|
35
|
+
@wake: Thread::ConditionVariable
|
|
36
|
+
@on_stop_request: (^() -> void)?
|
|
37
|
+
# nil until #run
|
|
38
|
+
@started_at: Integer
|
|
39
|
+
@heartbeat: Thread
|
|
40
|
+
@chores: Thread
|
|
41
|
+
@acker: Thread
|
|
42
|
+
# nil unless config.health_port is set
|
|
43
|
+
@health: HealthServer
|
|
44
|
+
# nil until the first heartbeat
|
|
45
|
+
@beat_at: Float
|
|
46
|
+
@beaten: bool?
|
|
47
|
+
@rtt: Integer
|
|
48
|
+
@collected_counts: Hash[Processor, Integer]?
|
|
49
|
+
@rss: Integer
|
|
50
|
+
@rss_at: Float
|
|
51
|
+
|
|
52
|
+
attr_reader config: Config
|
|
53
|
+
|
|
54
|
+
# This process's identity: "hostname:pid:random".
|
|
55
|
+
attr_reader identity: String
|
|
56
|
+
|
|
57
|
+
attr_reader managers: Array[Manager]
|
|
58
|
+
|
|
59
|
+
attr_reader collector: Metrics::Collector
|
|
60
|
+
|
|
61
|
+
attr_reader leader: Leader
|
|
62
|
+
|
|
63
|
+
# When it started running (epoch ms), or nil before #run.
|
|
64
|
+
def started_at: () -> Integer?
|
|
65
|
+
|
|
66
|
+
# Called (instead of stopping on a thread of its own) when the dashboard
|
|
67
|
+
# asks the process to stop.
|
|
68
|
+
%a{pure}
|
|
69
|
+
def on_stop_request: () -> (^() -> void)?
|
|
70
|
+
|
|
71
|
+
def on_stop_request=: ((^() -> void)? callback) -> (^() -> void)?
|
|
72
|
+
|
|
73
|
+
def initialize: (Config config, ?embedded: bool) -> void
|
|
74
|
+
|
|
75
|
+
def quiet?: () -> bool
|
|
76
|
+
|
|
77
|
+
# How many workers it runs, in every capsule.
|
|
78
|
+
def concurrency: () -> Integer
|
|
79
|
+
|
|
80
|
+
# Has capsule +capsule+ run +count+ workers from now on; false (changing
|
|
81
|
+
# nothing) when the process is quiet or the Redis pool can't grow.
|
|
82
|
+
def resize: (Integer count, ?capsule: String) -> bool
|
|
83
|
+
|
|
84
|
+
def stopping?: () -> bool
|
|
85
|
+
|
|
86
|
+
# Runs the block on every heartbeat.
|
|
87
|
+
def every_beat: () { (Redis::Connection conn) -> void } -> Array[hook]
|
|
88
|
+
|
|
89
|
+
# Runs the block on every heartbeat, on the leader only.
|
|
90
|
+
def every_leader_beat: () { (Redis::Connection conn) -> void } -> Array[hook]
|
|
91
|
+
|
|
92
|
+
def run: () -> self
|
|
93
|
+
|
|
94
|
+
# Stop fetching new jobs; running jobs finish.
|
|
95
|
+
def quiet: () -> void
|
|
96
|
+
|
|
97
|
+
# Graceful shutdown.
|
|
98
|
+
def stop: () -> void
|
|
99
|
+
|
|
100
|
+
# This process's health, as Robin::HealthServer serves it.
|
|
101
|
+
def health: () -> Hash[String, bool | String | Float | Integer | nil]
|
|
102
|
+
|
|
103
|
+
def inflight_key: () -> String
|
|
104
|
+
|
|
105
|
+
def proc_key: () -> String
|
|
106
|
+
|
|
107
|
+
def stopping_key: () -> String
|
|
108
|
+
|
|
109
|
+
def work_started: (Processor processor, Fetcher::UnitOfWork unit) -> void
|
|
110
|
+
|
|
111
|
+
def work_finished: (Processor processor, job_hash payload, Float | Integer elapsed_ms, bool ok) -> void
|
|
112
|
+
|
|
113
|
+
def work_finished_cleanup: (Processor processor) -> void
|
|
114
|
+
|
|
115
|
+
def busy: () -> Integer
|
|
116
|
+
|
|
117
|
+
# [processor, its work] for every job running now.
|
|
118
|
+
def running: () -> Array[[Processor, Processor::work]]
|
|
119
|
+
|
|
120
|
+
# The process's record, as the dashboard shows it (JSON).
|
|
121
|
+
def process_info: () -> Hash[String, json]
|
|
122
|
+
|
|
123
|
+
# One heartbeat, in one round trip.
|
|
124
|
+
def beat: () -> void
|
|
125
|
+
|
|
126
|
+
# Runs a full GC once the process has gone idle (see Config#idle_gc).
|
|
127
|
+
def collect_idle_garbage: () -> void
|
|
128
|
+
|
|
129
|
+
def collect_garbage: () -> void
|
|
130
|
+
|
|
131
|
+
# Puts back the jobs fetched for workers that are from a paused queue
|
|
132
|
+
# or that no worker took in time.
|
|
133
|
+
def hand_back_stale: (Redis::Connection conn, Array[String] paused) -> void
|
|
134
|
+
|
|
135
|
+
# The process title of a standalone server.
|
|
136
|
+
def title: () -> String
|
|
137
|
+
|
|
138
|
+
# Handles a remote signal ("TSTP", "TERM", "TTIN", "RESIZE <n> <capsule>").
|
|
139
|
+
def handle_signal: (String signal) -> void
|
|
140
|
+
|
|
141
|
+
def request_stop: () -> void
|
|
142
|
+
|
|
143
|
+
def dump_threads: () -> void
|
|
144
|
+
|
|
145
|
+
private
|
|
146
|
+
|
|
147
|
+
def work_json: (String queue, String raw, Integer run_at) -> String
|
|
148
|
+
|
|
149
|
+
def check!: () -> void
|
|
150
|
+
|
|
151
|
+
def connected: (Redis::Connection conn, String version) -> String
|
|
152
|
+
|
|
153
|
+
def start_health_server: () -> void
|
|
154
|
+
|
|
155
|
+
def mark_stopping: () -> void
|
|
156
|
+
|
|
157
|
+
def housekeeping_thread: (String name) { () -> void } -> Thread
|
|
158
|
+
|
|
159
|
+
def nap: (seconds seconds) -> void
|
|
160
|
+
|
|
161
|
+
def wake_up: () -> void
|
|
162
|
+
|
|
163
|
+
def send_acks: () -> void
|
|
164
|
+
|
|
165
|
+
def leader_chores: () -> void
|
|
166
|
+
|
|
167
|
+
def warn_about_slow_redis: (Integer rtt_us) -> void
|
|
168
|
+
|
|
169
|
+
def check_gvl: (Float percent) -> void
|
|
170
|
+
|
|
171
|
+
def warn_about_lapse: () -> void
|
|
172
|
+
|
|
173
|
+
def held_keys: () -> Array[String]
|
|
174
|
+
|
|
175
|
+
def sweep_inflight: (Redis::Connection conn, Array[String] registered) -> void
|
|
176
|
+
|
|
177
|
+
def every: (Integer seconds, Symbol name) { () -> void } -> void
|
|
178
|
+
|
|
179
|
+
def due?: (Integer seconds, Symbol name) -> (Float | false)
|
|
180
|
+
|
|
181
|
+
def prune_processes: (Redis::Connection conn) -> Redis::reply
|
|
182
|
+
|
|
183
|
+
def check_memory: () -> void
|
|
184
|
+
|
|
185
|
+
def rss_kb: () -> Integer
|
|
186
|
+
|
|
187
|
+
def safely: (String what) { () -> void } -> void
|
|
188
|
+
|
|
189
|
+
def monotonic: () -> Float
|
|
190
|
+
end
|
|
191
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Cluster-wide leader election using a Redis lease.
|
|
3
|
+
class Leader
|
|
4
|
+
TTL_MS: Integer
|
|
5
|
+
|
|
6
|
+
self.@leaders: Hash[Integer, Leader]
|
|
7
|
+
|
|
8
|
+
# Whether this process currently leads the cluster behind +config+.
|
|
9
|
+
def self.leader?: (?Config config) -> bool
|
|
10
|
+
|
|
11
|
+
def self.register: (Config config, Leader elector) -> Leader
|
|
12
|
+
|
|
13
|
+
def self.unregister: (Config config) -> Leader?
|
|
14
|
+
|
|
15
|
+
# Identity of the current leader, from Redis.
|
|
16
|
+
def self.current: (?Config config) -> String?
|
|
17
|
+
|
|
18
|
+
@config: Config
|
|
19
|
+
@leader: bool
|
|
20
|
+
@until: Float | Integer
|
|
21
|
+
# nil until #lease
|
|
22
|
+
@asked: Float
|
|
23
|
+
|
|
24
|
+
attr_reader identity: String
|
|
25
|
+
|
|
26
|
+
def initialize: (Config config, String identity) -> void
|
|
27
|
+
|
|
28
|
+
def leader?: () -> bool
|
|
29
|
+
|
|
30
|
+
# Queues a request to acquire or renew the lease on +pipe+; pass its
|
|
31
|
+
# reply to #settle.
|
|
32
|
+
def lease: (Redis::Pipeline pipe) -> nil
|
|
33
|
+
|
|
34
|
+
# Takes the reply to #lease; returns whether this process leads.
|
|
35
|
+
def settle: (Redis::reply reply) -> bool
|
|
36
|
+
|
|
37
|
+
def resign: (Redis::Connection conn) -> void
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def monotonic: () -> Float
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Distributed rate limiting, usable from jobs or any other Ruby code.
|
|
3
|
+
module Limiter
|
|
4
|
+
# An interval: :second, :minute, :hour or :day (or as a String), a
|
|
5
|
+
# number of seconds, or nil.
|
|
6
|
+
type interval = :second | :minute | :hour | :day | String | Integer | Float | nil
|
|
7
|
+
|
|
8
|
+
# What #within_limit does when it can't get capacity in time.
|
|
9
|
+
type policy = :raise | :skip
|
|
10
|
+
|
|
11
|
+
# A limiter's +backoff:+: the seconds to wait before running a
|
|
12
|
+
# rate-limited job again, or nil for the default.
|
|
13
|
+
interface _Backoff
|
|
14
|
+
def call: (Integer count, OverLimit error, job_hash payload) -> seconds?
|
|
15
|
+
|
|
16
|
+
include _Object
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# A limiter's +backoff:+ (see Config#rate_limit_backoff).
|
|
20
|
+
type backoff = ^(Integer count, OverLimit error, job_hash payload) -> seconds? | _Backoff
|
|
21
|
+
|
|
22
|
+
# The options every kind of limiter takes.
|
|
23
|
+
type options = { ?wait_timeout: seconds, ?reschedule: Integer?, ?backoff: backoff?, ?policy: policy, ?config: Config? }
|
|
24
|
+
|
|
25
|
+
# Raised by #within_limit when capacity didn't free up in time.
|
|
26
|
+
class OverLimit < Robin::Error
|
|
27
|
+
attr_reader limiter: Base
|
|
28
|
+
|
|
29
|
+
# Seconds until the limiter expects capacity, when it can tell.
|
|
30
|
+
attr_reader retry_after: Float?
|
|
31
|
+
|
|
32
|
+
def initialize: (Base limiter, ?Float? retry_after) -> void
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
COUNT_OVER: String
|
|
36
|
+
|
|
37
|
+
# A limiter's acquiring script, from +body+.
|
|
38
|
+
def self.acquiring: (String body) -> Redis::Script
|
|
39
|
+
|
|
40
|
+
INTERVALS: Hash[Symbol, Integer]
|
|
41
|
+
TYPES: Hash[String, String]
|
|
42
|
+
META_TTL: Integer
|
|
43
|
+
REGISTER_EVERY: Integer
|
|
44
|
+
|
|
45
|
+
# {[pool id, meta key] => [[type, limit, interval], monotonic time]}.
|
|
46
|
+
self.@registered: Hash[[Integer, String], [[String, Integer, Integer | Float | nil], Float]]
|
|
47
|
+
|
|
48
|
+
def self.registered: () -> Hash[[Integer, String], [[String, Integer, Integer | Float | nil], Float]]
|
|
49
|
+
|
|
50
|
+
# Sends every limiter call this thread makes inside the block to +pool+.
|
|
51
|
+
def self.via: [T] (Redis::Pool? pool) { () -> T } -> T
|
|
52
|
+
|
|
53
|
+
# The pool set by an enclosing Limiter.via block, if any.
|
|
54
|
+
def self.current_pool: () -> Redis::Pool?
|
|
55
|
+
|
|
56
|
+
# At most +size+ holders at once.
|
|
57
|
+
def self.concurrent: (_ToS name, Integer | String size, ?wait_timeout: seconds, ?lock_timeout: seconds, ?reschedule: Integer?, ?backoff: backoff?, ?policy: policy, ?config: Config?) -> Concurrent
|
|
58
|
+
|
|
59
|
+
# +count+ uses per calendar +interval+.
|
|
60
|
+
def self.bucket: (_ToS name, Integer | String count, interval interval, ?wait_timeout: seconds, ?reschedule: Integer?, ?backoff: backoff?, ?policy: policy, ?config: Config?) -> Bucket
|
|
61
|
+
|
|
62
|
+
# At most +count+ uses in any rolling +interval+.
|
|
63
|
+
def self.window: (_ToS name, Integer | String count, interval interval, ?wait_timeout: seconds, ?reschedule: Integer?, ?backoff: backoff?, ?policy: policy, ?config: Config?) -> Window
|
|
64
|
+
|
|
65
|
+
# Bursts up to +size+, draining at size/interval per second.
|
|
66
|
+
def self.leaky: (_ToS name, Integer | String size, interval interval, ?wait_timeout: seconds, ?reschedule: Integer?, ?backoff: backoff?, ?policy: policy, ?config: Config?) -> Leaky
|
|
67
|
+
|
|
68
|
+
# A token bucket of +points+, refilled over +interval+.
|
|
69
|
+
def self.points: (_ToS name, Integer | String points, interval interval, ?wait_timeout: seconds, ?reschedule: Integer?, ?backoff: backoff?, ?policy: policy, ?config: Config?) -> Points
|
|
70
|
+
|
|
71
|
+
# A limiter that never limits.
|
|
72
|
+
def self.unlimited: (?_ToS name) -> Unlimited
|
|
73
|
+
|
|
74
|
+
# Seconds for an interval, or nil.
|
|
75
|
+
def self.seconds: (interval interval) -> (Integer | Float | nil)
|
|
76
|
+
|
|
77
|
+
# A limiter as Limiter.list shows it: "name", "type", "limit",
|
|
78
|
+
# "interval", "used", "over_limit_count", "last_used_at".
|
|
79
|
+
type listing = Hash[String, untyped]
|
|
80
|
+
|
|
81
|
+
# Every limiter used in the last 30 days, with current usage.
|
|
82
|
+
def self.list: (?config: Config) -> Array[listing]
|
|
83
|
+
|
|
84
|
+
# Clears a limiter's state and over-limit counter.
|
|
85
|
+
def self.reset!: (String name, ?config: Config) -> true
|
|
86
|
+
|
|
87
|
+
# Forgets a limiter: its state and its definition.
|
|
88
|
+
def self.delete!: (String name, ?config: Config) -> true
|
|
89
|
+
|
|
90
|
+
private def self.clear: (String name, Config config) { (Redis::Pipeline p, bool known) -> void } -> void
|
|
91
|
+
|
|
92
|
+
private def self.from_meta: (String name, Hash[String, String] meta, Config config) -> Base?
|
|
93
|
+
|
|
94
|
+
private def self.interval_from: (Hash[String, String] meta) -> Float?
|
|
95
|
+
|
|
96
|
+
# What every limiter (but Unlimited) shares.
|
|
97
|
+
class Base
|
|
98
|
+
POLICIES: Array[Symbol]
|
|
99
|
+
|
|
100
|
+
@config: Config?
|
|
101
|
+
|
|
102
|
+
attr_reader name: String
|
|
103
|
+
attr_reader limit: Integer
|
|
104
|
+
|
|
105
|
+
# Seconds (nil for limiters without an interval).
|
|
106
|
+
attr_reader interval: Integer | Float | nil
|
|
107
|
+
|
|
108
|
+
# Milliseconds (nil until a subclass checks its interval).
|
|
109
|
+
attr_reader interval_ms: Integer?
|
|
110
|
+
|
|
111
|
+
attr_reader wait_timeout: Float
|
|
112
|
+
|
|
113
|
+
# How many times a job is rescheduled before OverLimit fails it.
|
|
114
|
+
attr_reader reschedule: Integer?
|
|
115
|
+
|
|
116
|
+
attr_reader backoff: backoff?
|
|
117
|
+
|
|
118
|
+
attr_reader policy: policy
|
|
119
|
+
|
|
120
|
+
def initialize: (_ToS name, Integer | String limit, interval interval, ?wait_timeout: seconds, ?reschedule: Integer | String | nil, ?backoff: backoff?, ?policy: policy, ?config: Config?) -> void
|
|
121
|
+
|
|
122
|
+
def config: () -> Config
|
|
123
|
+
|
|
124
|
+
# "concurrent", "bucket", "window", "leaky" or "points".
|
|
125
|
+
def type: () -> String
|
|
126
|
+
|
|
127
|
+
# Runs the block once capacity is available; returns what it returns
|
|
128
|
+
# (nil, without running it, when skipping).
|
|
129
|
+
def within_limit: [T] (**Integer | Float opts) { (self limiter) -> T } -> T?
|
|
130
|
+
|
|
131
|
+
def over_limit!: () -> Redis::reply
|
|
132
|
+
|
|
133
|
+
def reset_state: () -> Redis::reply
|
|
134
|
+
|
|
135
|
+
# The command #reset_state sends.
|
|
136
|
+
def reset_command: () -> Redis::command
|
|
137
|
+
|
|
138
|
+
# The command reading what #used reports.
|
|
139
|
+
def used_command: () -> Redis::command
|
|
140
|
+
|
|
141
|
+
# How many uses are counted now.
|
|
142
|
+
def used: () -> Integer
|
|
143
|
+
|
|
144
|
+
# What the reply to #used_command means.
|
|
145
|
+
def used_from: (Redis::reply reply) -> Integer
|
|
146
|
+
|
|
147
|
+
def to_h: () -> Hash[String, untyped]
|
|
148
|
+
|
|
149
|
+
private
|
|
150
|
+
|
|
151
|
+
def meta_key: () -> String
|
|
152
|
+
|
|
153
|
+
def state_key: () -> String
|
|
154
|
+
|
|
155
|
+
def now_ms: () -> Integer
|
|
156
|
+
|
|
157
|
+
def release: (bool | String token) -> void
|
|
158
|
+
|
|
159
|
+
# Takes capacity: [took it?, wait (ms)]. (Concurrent's returns its
|
|
160
|
+
# token or false, which unpack the same way, with no wait.)
|
|
161
|
+
def acquire: (wait: [Float, Float], **untyped) -> [bool | String, Integer?]
|
|
162
|
+
|
|
163
|
+
# [took capacity?, wait (ms)] for a script's reply.
|
|
164
|
+
def outcome: (Integer wait) -> [bool, Integer]
|
|
165
|
+
|
|
166
|
+
def wait_for_capacity: (Hash[Symbol, Integer | Float] opts) -> (bool | String | nil)
|
|
167
|
+
|
|
168
|
+
def register: () -> void
|
|
169
|
+
|
|
170
|
+
def monotonic: () -> Float
|
|
171
|
+
|
|
172
|
+
def pool: () -> Redis::Pool
|
|
173
|
+
|
|
174
|
+
def redis: [T] () { (Redis::Connection conn) -> T } -> T
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# Lets one call count as several uses (+used:+).
|
|
178
|
+
module Uses : Base
|
|
179
|
+
def within_limit: [T] (?used: Integer) { (Base limiter) -> T } -> T?
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# At most +size+ holders at once, each leasing its slot for
|
|
183
|
+
# +lock_timeout+ seconds.
|
|
184
|
+
class Concurrent < Base
|
|
185
|
+
ACQUIRE: Redis::Script
|
|
186
|
+
|
|
187
|
+
attr_reader lock_timeout: Float
|
|
188
|
+
attr_reader lock_timeout_ms: Integer
|
|
189
|
+
|
|
190
|
+
# Other options: see Base#initialize.
|
|
191
|
+
def initialize: (_ToS name, Integer | String size, ?interval _interval, ?lock_timeout: seconds, **untyped options) -> void
|
|
192
|
+
|
|
193
|
+
def within_limit: [T] () { (self limiter) -> T } -> T?
|
|
194
|
+
|
|
195
|
+
private
|
|
196
|
+
|
|
197
|
+
def acquire: (wait: [Float, Float], **untyped) -> (String | false)
|
|
198
|
+
|
|
199
|
+
def release: (String token) -> void
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# +limit+ uses per calendar interval.
|
|
203
|
+
class Bucket < Base
|
|
204
|
+
include Uses
|
|
205
|
+
|
|
206
|
+
ACQUIRE: Redis::Script
|
|
207
|
+
|
|
208
|
+
def initialize: (_ToS name, Integer | String limit, interval interval, **untyped options) -> void
|
|
209
|
+
|
|
210
|
+
# Milliseconds (every limiter of this kind has an interval).
|
|
211
|
+
def interval_ms: () -> Integer
|
|
212
|
+
|
|
213
|
+
def within_limit: [T] (?used: Integer) { (self limiter) -> T } -> T?
|
|
214
|
+
|
|
215
|
+
private
|
|
216
|
+
|
|
217
|
+
def bucket_key: (Integer now) -> String
|
|
218
|
+
|
|
219
|
+
def acquire: (used: Integer, wait: [Float, Float]) -> [bool, Integer]
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# At most +limit+ uses in any rolling +interval+.
|
|
223
|
+
class Window < Base
|
|
224
|
+
include Uses
|
|
225
|
+
|
|
226
|
+
ACQUIRE: Redis::Script
|
|
227
|
+
|
|
228
|
+
def initialize: (_ToS name, Integer | String limit, interval interval, **untyped options) -> void
|
|
229
|
+
|
|
230
|
+
# Milliseconds (every limiter of this kind has an interval).
|
|
231
|
+
def interval_ms: () -> Integer
|
|
232
|
+
|
|
233
|
+
def within_limit: [T] (?used: Integer) { (self limiter) -> T } -> T?
|
|
234
|
+
|
|
235
|
+
private
|
|
236
|
+
|
|
237
|
+
def acquire: (used: Integer, wait: [Float, Float]) -> [bool, Integer]
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
# Capacity +limit+, draining at limit/interval per second.
|
|
241
|
+
class Leaky < Base
|
|
242
|
+
ACQUIRE: Redis::Script
|
|
243
|
+
|
|
244
|
+
def initialize: (_ToS name, Integer | String limit, interval interval, **untyped options) -> void
|
|
245
|
+
|
|
246
|
+
# Milliseconds (every limiter of this kind has an interval).
|
|
247
|
+
def interval_ms: () -> Integer
|
|
248
|
+
|
|
249
|
+
def within_limit: [T] () { (self limiter) -> T } -> T?
|
|
250
|
+
|
|
251
|
+
def used_from: (Array[String?] reply) -> Integer
|
|
252
|
+
|
|
253
|
+
private
|
|
254
|
+
|
|
255
|
+
def acquire: (wait: [Float, Float], **untyped) -> [bool, Integer]
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# A token bucket of +limit+ points, refilled over +interval+.
|
|
259
|
+
class Points < Base
|
|
260
|
+
ACQUIRE: Redis::Script
|
|
261
|
+
ADJUST: Redis::Script
|
|
262
|
+
|
|
263
|
+
def initialize: (_ToS name, Integer | String limit, interval interval, **untyped options) -> void
|
|
264
|
+
|
|
265
|
+
# Milliseconds (every limiter of this kind has an interval).
|
|
266
|
+
def interval_ms: () -> Integer
|
|
267
|
+
|
|
268
|
+
def within_limit: [T] (?points: Integer | Float) { (self limiter) -> T } -> T?
|
|
269
|
+
|
|
270
|
+
# Spends (positive) or returns (negative) points; returns the points
|
|
271
|
+
# now available.
|
|
272
|
+
def adjust: (Integer | Float points) -> Float
|
|
273
|
+
|
|
274
|
+
def refund: (Integer | Float points) -> Float
|
|
275
|
+
|
|
276
|
+
def used_from: (Array[String?] reply) -> Integer
|
|
277
|
+
|
|
278
|
+
private
|
|
279
|
+
|
|
280
|
+
def acquire: (points: Integer | Float, wait: [Float, Float]) -> [bool, Integer]
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# A limiter that never limits (and never touches Redis).
|
|
284
|
+
class Unlimited
|
|
285
|
+
attr_reader name: String
|
|
286
|
+
|
|
287
|
+
def initialize: (_ToS name) -> void
|
|
288
|
+
|
|
289
|
+
def type: () -> String
|
|
290
|
+
|
|
291
|
+
def within_limit: [T] (**Integer | Float) { (self limiter) -> T } -> T
|
|
292
|
+
|
|
293
|
+
def used: () -> Integer
|
|
294
|
+
|
|
295
|
+
def adjust: (Integer | Float _points) -> Float
|
|
296
|
+
|
|
297
|
+
def refund: (Integer | Float _points) -> Float
|
|
298
|
+
|
|
299
|
+
def to_h: () -> Hash[String, untyped]
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# Reschedules jobs whose limiter is over its limit instead of failing
|
|
303
|
+
# them.
|
|
304
|
+
class ServerMiddleware
|
|
305
|
+
include Middleware::ConfigAware
|
|
306
|
+
include Middleware::Stateless
|
|
307
|
+
|
|
308
|
+
def call: (untyped job, job_hash payload, String queue) { () -> untyped } -> untyped
|
|
309
|
+
|
|
310
|
+
private
|
|
311
|
+
|
|
312
|
+
def custom_delay: (OverLimit error, Integer count, job_hash payload) -> Numeric?
|
|
313
|
+
|
|
314
|
+
def default_delay: (OverLimit error, Integer count, job_hash payload) -> Float
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Owns the worker threads of one capsule.
|
|
3
|
+
class Manager
|
|
4
|
+
RESPAWN_WINDOW: Integer
|
|
5
|
+
|
|
6
|
+
MAX_RESPAWN_DELAY: Integer
|
|
7
|
+
|
|
8
|
+
@config: Config
|
|
9
|
+
@mutex: Thread::Mutex
|
|
10
|
+
@slots: Integer
|
|
11
|
+
@processors: Array[Processor]
|
|
12
|
+
@stopping: bool?
|
|
13
|
+
@started: bool?
|
|
14
|
+
@deaths: Integer
|
|
15
|
+
# nil until a worker dies
|
|
16
|
+
@last_death: Float
|
|
17
|
+
|
|
18
|
+
attr_reader capsule: Capsule
|
|
19
|
+
attr_reader fetcher: Fetcher
|
|
20
|
+
attr_reader launcher: Launcher
|
|
21
|
+
|
|
22
|
+
def initialize: (Capsule capsule, Launcher launcher) -> void
|
|
23
|
+
|
|
24
|
+
# How many workers it runs.
|
|
25
|
+
def concurrency: () -> Integer
|
|
26
|
+
|
|
27
|
+
# Runs +count+ workers from now on.
|
|
28
|
+
def resize: (Integer count) -> void
|
|
29
|
+
|
|
30
|
+
def start: () -> void
|
|
31
|
+
|
|
32
|
+
def processors: () -> Array[Processor]
|
|
33
|
+
|
|
34
|
+
def quiet: () -> void
|
|
35
|
+
|
|
36
|
+
# Waits for running jobs until +deadline+ (monotonic seconds), then
|
|
37
|
+
# interrupts the rest and waits for them until +hard_deadline+.
|
|
38
|
+
def stop: (Float deadline, Float hard_deadline) -> void
|
|
39
|
+
|
|
40
|
+
def processor_stopped: (Processor processor) -> void
|
|
41
|
+
|
|
42
|
+
def processor_died: (Processor processor, Exception exception) -> void
|
|
43
|
+
|
|
44
|
+
def work_started: (Processor processor, Fetcher::UnitOfWork unit) -> void
|
|
45
|
+
|
|
46
|
+
def work_finished: (Processor processor, job_hash payload, Float | Integer elapsed_ms, bool ok) -> void
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def respawn_delay: () -> Integer
|
|
51
|
+
|
|
52
|
+
def monotonic: () -> Float
|
|
53
|
+
end
|
|
54
|
+
end
|