robin 0.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/java/robin_resp/Blocking.java +27 -0
  13. data/ext/java/robin_resp/NativeIO.java +229 -0
  14. data/ext/java/robin_resp/NativeRESP.java +279 -0
  15. data/ext/java/robin_resp/NativeSlots.java +133 -0
  16. data/ext/java/robin_resp/RobinRespService.java +32 -0
  17. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  18. data/lib/generators/robin/install/install_generator.rb +39 -0
  19. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  20. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  21. data/lib/generators/robin/job/job_generator.rb +78 -0
  22. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  23. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  24. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  25. data/lib/puma/plugin/robin.rb +131 -0
  26. data/lib/robin/active_job.rb +144 -0
  27. data/lib/robin/api.rb +777 -0
  28. data/lib/robin/batch.rb +700 -0
  29. data/lib/robin/cancellation.rb +62 -0
  30. data/lib/robin/cli.rb +325 -0
  31. data/lib/robin/client.rb +372 -0
  32. data/lib/robin/config.rb +461 -0
  33. data/lib/robin/cron.rb +325 -0
  34. data/lib/robin/debounce.rb +108 -0
  35. data/lib/robin/deploy.rb +65 -0
  36. data/lib/robin/embedded.rb +53 -0
  37. data/lib/robin/encryption.rb +147 -0
  38. data/lib/robin/events.rb +60 -0
  39. data/lib/robin/expiry.rb +41 -0
  40. data/lib/robin/fetcher.rb +441 -0
  41. data/lib/robin/gvl.rb +62 -0
  42. data/lib/robin/health.rb +76 -0
  43. data/lib/robin/iterable_job.rb +294 -0
  44. data/lib/robin/job.rb +258 -0
  45. data/lib/robin/job_class.rb +157 -0
  46. data/lib/robin/job_logger.rb +121 -0
  47. data/lib/robin/launcher.rb +658 -0
  48. data/lib/robin/leader.rb +86 -0
  49. data/lib/robin/limiter.rb +639 -0
  50. data/lib/robin/manager.rb +125 -0
  51. data/lib/robin/metrics.rb +303 -0
  52. data/lib/robin/middleware.rb +165 -0
  53. data/lib/robin/opentelemetry.rb +88 -0
  54. data/lib/robin/periodic.rb +327 -0
  55. data/lib/robin/processor.rb +317 -0
  56. data/lib/robin/profiler.rb +140 -0
  57. data/lib/robin/progress.rb +46 -0
  58. data/lib/robin/rails/app_loader.rb +23 -0
  59. data/lib/robin/rails/current_attributes.rb +88 -0
  60. data/lib/robin/rails/error_reporter.rb +31 -0
  61. data/lib/robin/rails/notifications.rb +22 -0
  62. data/lib/robin/rails/railtie.rb +87 -0
  63. data/lib/robin/rails/transactional_push.rb +49 -0
  64. data/lib/robin/rails.rb +23 -0
  65. data/lib/robin/recovery.rb +135 -0
  66. data/lib/robin/redis/connection.rb +639 -0
  67. data/lib/robin/redis/pool.rb +190 -0
  68. data/lib/robin/redis/resp.rb +77 -0
  69. data/lib/robin/reliable_push.rb +153 -0
  70. data/lib/robin/retrier.rb +284 -0
  71. data/lib/robin/scheduler.rb +102 -0
  72. data/lib/robin/scripts.rb +331 -0
  73. data/lib/robin/statsd.rb +135 -0
  74. data/lib/robin/swarm.rb +319 -0
  75. data/lib/robin/systemd.rb +47 -0
  76. data/lib/robin/testing.rb +158 -0
  77. data/lib/robin/top.rb +178 -0
  78. data/lib/robin/unique.rb +198 -0
  79. data/lib/robin/version.rb +5 -0
  80. data/lib/robin/warmup.rb +94 -0
  81. data/lib/robin/web/api.rb +580 -0
  82. data/lib/robin/web/batch_status.rb +44 -0
  83. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  84. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  86. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  87. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  88. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  89. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  90. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  91. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  92. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  93. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  94. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  95. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  96. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  97. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  98. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  99. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  100. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  101. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  102. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  103. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  104. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  105. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  106. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  107. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  108. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  109. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  110. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  111. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  112. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  113. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  114. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  115. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  116. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  117. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  118. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  119. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  120. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  121. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  122. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  123. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  124. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  125. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  126. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  127. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  128. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  129. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  130. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  131. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  132. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  133. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  134. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  135. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  136. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  137. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  138. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  139. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  140. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  141. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  142. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  143. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  144. data/lib/robin/web/dist/index.html +19 -0
  145. data/lib/robin/web/extension.rb +193 -0
  146. data/lib/robin/web/permissions.rb +40 -0
  147. data/lib/robin/web/prometheus.rb +79 -0
  148. data/lib/robin/web/router.rb +150 -0
  149. data/lib/robin/web/static.rb +110 -0
  150. data/lib/robin/web.rb +196 -0
  151. data/lib/robin.rb +261 -0
  152. data/lib/robin_resp/robin_resp.jar +0 -0
  153. data/sig/manifest.yaml +9 -0
  154. data/sig/robin/api.rbs +442 -0
  155. data/sig/robin/batch.rbs +292 -0
  156. data/sig/robin/cancellation.rbs +25 -0
  157. data/sig/robin/cli.rbs +102 -0
  158. data/sig/robin/client.rbs +118 -0
  159. data/sig/robin/config.rbs +341 -0
  160. data/sig/robin/cron.rbs +149 -0
  161. data/sig/robin/debounce.rbs +36 -0
  162. data/sig/robin/deploy.rbs +24 -0
  163. data/sig/robin/embedded.rbs +25 -0
  164. data/sig/robin/encryption.rbs +49 -0
  165. data/sig/robin/events.rbs +10 -0
  166. data/sig/robin/expiry.rbs +17 -0
  167. data/sig/robin/fetcher.rbs +134 -0
  168. data/sig/robin/gvl.rbs +39 -0
  169. data/sig/robin/health.rbs +30 -0
  170. data/sig/robin/iterable_job.rbs +99 -0
  171. data/sig/robin/job.rbs +173 -0
  172. data/sig/robin/job_class.rbs +55 -0
  173. data/sig/robin/job_logger.rbs +56 -0
  174. data/sig/robin/launcher.rbs +191 -0
  175. data/sig/robin/leader.rbs +43 -0
  176. data/sig/robin/limiter.rbs +317 -0
  177. data/sig/robin/manager.rbs +54 -0
  178. data/sig/robin/metrics.rbs +122 -0
  179. data/sig/robin/middleware.rbs +106 -0
  180. data/sig/robin/opentelemetry.rbs +51 -0
  181. data/sig/robin/periodic.rbs +81 -0
  182. data/sig/robin/processor.rbs +107 -0
  183. data/sig/robin/profiler.rbs +60 -0
  184. data/sig/robin/progress.rbs +22 -0
  185. data/sig/robin/rails/app_loader.rbs +10 -0
  186. data/sig/robin/rails/current_attributes.rbs +44 -0
  187. data/sig/robin/rails/error_reporter.rbs +21 -0
  188. data/sig/robin/rails/notifications.rbs +9 -0
  189. data/sig/robin/rails/transactional_push.rbs +16 -0
  190. data/sig/robin/rails.rbs +8 -0
  191. data/sig/robin/recovery.rbs +29 -0
  192. data/sig/robin/redis/connection.rbs +224 -0
  193. data/sig/robin/redis/pool.rbs +89 -0
  194. data/sig/robin/redis/resp.rbs +29 -0
  195. data/sig/robin/reliable_push.rbs +77 -0
  196. data/sig/robin/retrier.rbs +72 -0
  197. data/sig/robin/scheduler.rbs +40 -0
  198. data/sig/robin/scripts.rbs +36 -0
  199. data/sig/robin/statsd.rbs +55 -0
  200. data/sig/robin/swarm.rbs +133 -0
  201. data/sig/robin/systemd.rbs +13 -0
  202. data/sig/robin/testing.rbs +80 -0
  203. data/sig/robin/top.rbs +79 -0
  204. data/sig/robin/unique.rbs +72 -0
  205. data/sig/robin/version.rbs +3 -0
  206. data/sig/robin/warmup.rbs +44 -0
  207. data/sig/robin/web/api.rbs +196 -0
  208. data/sig/robin/web/batch_status.rbs +25 -0
  209. data/sig/robin/web/extension.rbs +106 -0
  210. data/sig/robin/web/permissions.rbs +14 -0
  211. data/sig/robin/web/prometheus.rbs +33 -0
  212. data/sig/robin/web/router.rbs +53 -0
  213. data/sig/robin/web/static.rbs +42 -0
  214. data/sig/robin/web.rbs +140 -0
  215. data/sig/robin.rbs +116 -0
  216. data/sig/robin_resp.rbs +55 -0
  217. metadata +294 -0
data/README.md ADDED
@@ -0,0 +1,1490 @@
1
+ <p align="center">
2
+ <img src="assets/robin-logo.svg" alt="Robin" width="420">
3
+ </p>
4
+
5
+ <p align="center"><strong>Fast, reliable background jobs for Ruby. Every feature open source.</strong></p>
6
+
7
+ <p align="center">
8
+ <a href="https://github.com/sferik/duel-ruby"><img src="https://raw.githubusercontent.com/sferik/robin/duel/badge/badge.svg" alt="Duel"></a>
9
+ <a href="https://github.com/sferik/robin/actions/workflows/test.yml"><img src="https://github.com/sferik/robin/actions/workflows/test.yml/badge.svg" alt="Test"></a>
10
+ <a href="https://github.com/sferik/robin/actions/workflows/lint.yml"><img src="https://github.com/sferik/robin/actions/workflows/lint.yml/badge.svg" alt="Lint"></a>
11
+ <a href="https://github.com/sferik/robin/actions/workflows/mutant.yml"><img src="https://github.com/sferik/robin/actions/workflows/mutant.yml/badge.svg" alt="Mutant"></a>
12
+ <a href="https://rubygems.org/gems/robin"><img src="https://badge.fury.io/rb/robin.svg" alt="Gem Version"></a>
13
+ </p>
14
+
15
+ Robin is a Redis-backed job system for Ruby. Batches, reliable fetch, rate limiting,
16
+ cron, unique jobs, encryption, multi-process swarms with rolling restarts,
17
+ historical metrics and a modern dashboard with multi-cluster support and
18
+ authorization hooks are all included under the LGPL-3.0. There is no paid tier.
19
+
20
+ ```ruby
21
+ class SendReceipt
22
+ include Robin::Job
23
+ robin_options queue: "mailers", retry: 5
24
+
25
+ def perform(order_id)
26
+ OrderMailer.receipt(Order.find(order_id)).deliver_now
27
+ end
28
+ end
29
+
30
+ SendReceipt.perform_async(order.id)
31
+ SendReceipt.perform_in(300, order.id) # in 5 minutes
32
+ SendReceipt.set(queue: "critical").perform_async(order.id)
33
+ ```
34
+
35
+ ```sh
36
+ bundle exec robin -r ./app.rb -c 10 -q critical,3 -q default
37
+ ```
38
+
39
+ ## Why Robin
40
+
41
+ **Faster.** Robin processes jobs 5x as fast as Sidekiq 8.1 OSS on the
42
+ same machine, 8.8x with Redis 1 ms away, and moves scheduled jobs onto
43
+ their queues 61x as fast, with an eighth of the Redis CPU per job
44
+ ([benchmarks](#performance)).
45
+
46
+ **Built not to lose jobs, or run them twice.**
47
+
48
+ - **A crash loses nothing.** A fetched job stays in Redis until it has
49
+ finished or its retry has been recorded. Kill a server with `kill -9` in
50
+ the middle of 1,000 jobs: Robin loses none, Sidekiq OSS loses every job
51
+ that was running ([crash benchmark](bench/RESULTS.md#surviving-a-crash)).
52
+ The cluster puts a dead process's jobs back on their queues by itself,
53
+ and a job that keeps killing its process goes to the dead set instead of
54
+ taking down the next one.
55
+ - **Each step happens once.** A due scheduled job or retry is moved by a
56
+ script that only queues it if it removed it from its set, so any number
57
+ of servers polling queue it once. Each cron tick is claimed with an
58
+ atomic `SET NX`, so it runs once across the cluster. A job's retry (or
59
+ death) is recorded in the same transaction that removes it from flight,
60
+ so a failure is never both retried and lost. A command is only resent
61
+ after a dropped connection if none of it reached Redis, so a failover
62
+ doesn't push a job twice.
63
+ - **Duplicates only where they can't be avoided.** Delivery is at least
64
+ once, as with any job system that survives crashes: a job runs again
65
+ only if its process dies while it runs (or within 2 ms of it finishing)
66
+ or a shutdown interrupts it. [Unique jobs](#unique-debounced-expiring-and-encrypted-jobs)
67
+ and idempotent jobs cover the rest.
68
+ - **Tested to the hilt.** About 1,900 tests run on CRuby 3.4, 4.0 and
69
+ JRuby. CI enforces 100% line, branch and method coverage of the Ruby, C,
70
+ Java and TypeScript code, and 100% mutation coverage of the Ruby, Java
71
+ and TypeScript code: every deliberate bug a mutation tester plants makes
72
+ some test fail ([details](#development)).
73
+
74
+ ## Contents
75
+
76
+ - [Why Robin](#why-robin)
77
+ - [Features](#features)
78
+ - [Installation](#installation)
79
+ - [Jobs](#jobs)
80
+ - [Running servers](#running-servers)
81
+ - [Reliability](#reliability)
82
+ - [Batches and workflows](#batches-and-workflows)
83
+ - [Rate limiting](#rate-limiting)
84
+ - [Periodic (cron) jobs](#periodic-cron-jobs)
85
+ - [Unique, debounced, expiring and encrypted jobs](#unique-debounced-expiring-and-encrypted-jobs)
86
+ - [Long-running (iterable) jobs](#long-running-iterable-jobs)
87
+ - [Progress and cancellation](#progress-and-cancellation)
88
+ - [Middleware](#middleware)
89
+ - [Errors, retries and the dead set](#errors-retries-and-the-dead-set)
90
+ - [Metrics](#metrics)
91
+ - [Tracing and profiling](#tracing-and-profiling)
92
+ - [Web dashboard](#web-dashboard)
93
+ - [Rails and ActiveJob](#rails-and-activejob)
94
+ - [Testing](#testing)
95
+ - [The data API](#the-data-api)
96
+ - [Performance](#performance)
97
+ - [Development](#development)
98
+ - [License](#license)
99
+
100
+ ## Features
101
+
102
+ | Area | What you get |
103
+ |---|---|
104
+ | Core | Multithreaded workers, capsules, strict or weighted queue priority, scheduled jobs, retries with exponential backoff (limited by count or by time, optionally on another queue), dead set, client and server middleware, strict JSON arguments, bulk push (optionally spread over time), embedded mode, deferrable pushes (transaction-aware in Rails), a Ruby API for everything in Redis |
105
+ | Reliability | Reliable fetch (in-flight jobs survive crashes), automatic recovery from dead processes, poison-pill detection, reliable push (buffers jobs while Redis is unreachable), queue pausing, and pausing a job class across every queue (Robin-only) |
106
+ | Workflows | Batches with `success`, `complete` and `death` callbacks, nested batches, tags for finding batches, cancelling a batch and its children, a status endpoint your own pages can poll |
107
+ | Throughput control | Concurrent, bucket, sliding-window, leaky-bucket and points rate limiters, each with its own reschedule count and backoff, calls that count as several uses, a policy that skips work instead of waiting, limiters on a Redis of their own, plus an unlimited stand-in; expiring jobs |
108
+ | Scheduling | Cron jobs (Robin jobs or ActiveJobs) with time zones, run exactly once by an elected leader, and a checker for your test suite |
109
+ | Deduplication | Unique jobs, and debounced jobs (Robin-only) |
110
+ | Security | AES-256-GCM argument encryption with key rotation, web UI authorization and read-only mode |
111
+ | Operations | Swarm mode (multi-process, copy-on-write preload, sized per CPU core), rolling restarts, per-process memory limits, leader election, remote quiet/stop/resize, `robin top` (a live view in the terminal) |
112
+ | Observability | Per-class timings and latency histograms, hourly history kept for 90 days by default, queue size and latency history, deploy markers on the charts, StatsD/DogStatsD (jobs, queues, batches, recovered jobs; Datadog distributions), Prometheus endpoint |
113
+ | Long jobs | Iterable jobs that checkpoint and resume across deploys (and yield their worker after a maximum runtime), live progress reporting, cooperative cancellation (Robin-only) |
114
+ | Dashboard | TypeScript single-page app in 25 languages (including right-to-left ones) with live updates, light and dark themes, search, editing a dead job's args before retrying it, multi-cluster support, rows of your own on job pages, pages of your own (built from blocks, or with your own script), and assets from a CDN |
115
+
116
+ ## Installation
117
+
118
+ ```ruby
119
+ # Gemfile
120
+ gem "robin"
121
+ ```
122
+
123
+ Robin needs Ruby 3.4+ (or JRuby 10) and Redis 6.2+, Valkey or Dragonfly.
124
+ CI runs every test on Ruby 3.4 and 4.0 (and Ruby's development version),
125
+ JRuby 10.1, Redis 6.2, 7.2, 7.4, 8.2 and 8.10, Valkey 9.1 and
126
+ Dragonfly 2.0. CI runs them on TruffleRuby 40 too, where Robin uses pure
127
+ Ruby, but TruffleRuby isn't supported yet: a few of its bugs fail tests
128
+ (a refused connection is reported as `EINVAL`, for one).
129
+ Dragonfly needs `--default_lua_flags=allow-undeclared-keys`, because some of
130
+ Robin's Lua scripts build key names from the key prefix. Robin has no runtime
131
+ dependencies besides `json` and `logger`: it ships its own small Redis
132
+ client. It works with any Ruby or Rack app, and in a Rails app its
133
+ [Rails integration](#rails-and-activejob) turns itself on. An optional C extension, compiled at
134
+ install time, reads and parses Redis replies faster. If it can't be built,
135
+ Robin falls back to pure Ruby automatically.
136
+
137
+ **Coming from Sidekiq?** Replace `gem "sidekiq"` (and `sidekiq-pro` or
138
+ `sidekiq-ent`) with `gem "robin-compat"`, and your app runs on Robin
139
+ unchanged, picking up Sidekiq's queued, scheduled, retrying and dead jobs
140
+ where they are. See [robin-compat](https://github.com/sferik/robin-compat).
141
+
142
+ On JRuby (10.1+), RubyGems installs Robin's java-platform gem instead. It
143
+ ships a compiled Java extension in place of the C one: the same reply parser
144
+ and socket reader, a command encoder, and lock-free bookkeeping for the
145
+ connection pool, so worker threads, which run in parallel on the JVM, never
146
+ wait on each other. Nothing needs compiling at install time. Give the JVM
147
+ a maximum heap (`JRUBY_OPTS=-J-Xmx512m`, say): without one, it grows the
148
+ heap under load toward a quarter of the machine's memory, though a busy
149
+ server's live heap is far smaller.
150
+
151
+ Point Robin at Redis with `ROBIN_REDIS_URL` or `REDIS_URL` (or
152
+ `REDIS_PROVIDER`, naming the variable that holds the URL, as some hosting
153
+ add-ons expect), or configure it:
154
+
155
+ ```ruby
156
+ Robin.configure do |config|
157
+ config.redis = {url: "redis://localhost:6379/0"} # also rediss://, unix://
158
+ end
159
+ ```
160
+
161
+ Other Redis options: `pool_size`, `pool_timeout`, `idle_timeout` (close
162
+ connections idle that many seconds), `name`, `connect_timeout`,
163
+ `read_timeout`, `write_timeout`, `ssl_params`.
164
+
165
+ With Redis Sentinel, name the master in the URL (or `master_name:`) and
166
+ list the Sentinels. Robin asks them where the master is on every new
167
+ connection, checks it really is the master, and moves to the new one after
168
+ a failover:
169
+
170
+ ```ruby
171
+ config.redis = {
172
+ url: "redis://mymaster/0",
173
+ sentinels: ["10.0.0.1:26379", "10.0.0.2:26379", {host: "10.0.0.3", port: 26379}],
174
+ sentinel_password: ENV["SENTINEL_PASSWORD"] # if the Sentinels need one
175
+ }
176
+ ```
177
+
178
+ To spread jobs over several Redis servers (shards), push through a pool for
179
+ the shard you want. Each shard needs its own Robin processes:
180
+
181
+ ```ruby
182
+ SHARDS = %w[redis://r1 redis://r2].map do |url|
183
+ Robin::Redis::Pool.new(size: 10) { Robin::Redis::Connection.new(url: url) }
184
+ end
185
+
186
+ ReportJob.set(pool: SHARDS.sample).perform_async(42)
187
+ Robin::Client.via(SHARDS[1]) { ReportJob.perform_async(1); OtherJob.perform_async(2) }
188
+ ```
189
+
190
+ ## Jobs
191
+
192
+ ```ruby
193
+ class ResizeImage
194
+ include Robin::Job
195
+ robin_options queue: "media",
196
+ retry: 10, # true (default: 20 retries), false, or a number
197
+ retry_for: 86_400, # or keep retrying for a day after the first failure
198
+ retry_queue: "slow", # retries run on this queue
199
+ backtrace: 20, # keep 20 backtrace lines on failure
200
+ tags: ["images"], # shown and searchable in the dashboard
201
+ timeout: 120, # last-resort hard timeout in seconds
202
+ log_level: :debug # level of its start/done lines (logging: false drops them)
203
+
204
+ def perform(image_id, width)
205
+ # ...
206
+ end
207
+ end
208
+
209
+ ResizeImage.perform_async(1, 640)
210
+ ResizeImage.perform_at(Time.now + 3600, 1, 640)
211
+ ResizeImage.set(queue: "urgent", wait: 30).perform_async(1, 640)
212
+ ResizeImage.perform_bulk([[1, 640], [2, 640], [3, 640]]) # pipelined
213
+ ResizeImage.perform_bulk(ids.map { [_1, 640] }, spread_interval: 3600) # evenly over the next hour
214
+ ResizeImage.perform_inline(1, 640) # run now, in this thread
215
+ Robin::Client.together { images.each { ResizeImage.perform_async(_1.id, 640) } } # one round trip
216
+ ```
217
+
218
+ `Robin::Client.together` holds back the jobs its block pushes and sends
219
+ them all when it finishes, in one round trip when they go straight to the
220
+ same Redis (none if the block raises).
221
+
222
+ Arguments must be JSON-native: strings, numbers, booleans, nil, arrays, and
223
+ hashes with string keys. Robin raises at push time if they aren't. Set
224
+ `config.strict_args = :warn` or `false` to relax this.
225
+
226
+ Raise `Robin::Discard` from a job to stop it without retrying.
227
+
228
+ `timeout:` raises `Robin::Job::Timeout` inside the job. It isn't a
229
+ `StandardError`, so a job's own `rescue => e` can't swallow it. Like any
230
+ `Timeout.timeout`, though, it can interrupt the job anywhere, even inside
231
+ an `ensure` block or halfway through a database call. Put timeouts on the
232
+ slow operations themselves (HTTP clients, queries) and keep this one as a
233
+ last resort.
234
+
235
+ ## Running servers
236
+
237
+ ```sh
238
+ robin -r ./app.rb -c 10 -q critical,3 -q default,1 # weighted
239
+ robin -r . -q critical -q default -q low # strict order (app directory)
240
+ robin -C config/robin.yml
241
+ robin swarm -n 4 # 4 processes, preloaded
242
+ robin swarm -n 0.5 # one process per two CPU cores
243
+ robin stats # cluster stats as JSON
244
+ robin top # live view of queues, processes and running jobs
245
+ robin deploy "v42" # mark a deploy on the metrics charts
246
+ robin health --health-port 7433 # exit 0 if this host's server is healthy
247
+ ```
248
+
249
+ ```yaml
250
+ # config/robin.yml
251
+ concurrency: 10
252
+ queues: [critical, default, low]
253
+ timeout: 25
254
+ capsules:
255
+ serial:
256
+ concurrency: 1
257
+ queues: [ledger]
258
+ production:
259
+ concurrency: 20
260
+ ```
261
+
262
+ `robin top` shows a live view of the cluster in the terminal: totals and
263
+ queues, processes, and the jobs running now, refreshed every 2 seconds.
264
+ 1, 2 and 3 (or tab) switch views, r refreshes and q quits. It only reads.
265
+ `robin stats`, `robin top` and `robin deploy` load the app as a client,
266
+ like a web process: its `configure_client` blocks run, not its
267
+ `configure_server` blocks.
268
+
269
+ `robin swarm -n` takes a number of processes, or a decimal meaning that
270
+ many per CPU core (`-n 1.5` on 8 cores starts 12). `ROBIN_SWARM_COUNT`
271
+ works too, and the default is one per core. Each child runs
272
+ `config.on(:fork)` hooks first, e.g. to reconnect to a database.
273
+
274
+ **Capsules** isolate work inside one process, each with its own threads and queues:
275
+
276
+ ```ruby
277
+ Robin.configure_server do |config|
278
+ config.capsule("serial") do |cap|
279
+ cap.concurrency = 1
280
+ cap.queues = %w[ledger]
281
+ end
282
+ end
283
+ ```
284
+
285
+ A job's `capsule` (or `Robin::Capsule.current`, anywhere on a worker's
286
+ thread) is the capsule running it, with its `name`, `queues` and
287
+ `concurrency`; it's nil outside a server's workers.
288
+
289
+ The process title shows the version, tag and how many jobs are running
290
+ (`robin 0.1.0 shop [3 of 10 busy]`). The tag defaults to the app
291
+ directory's name (the app's, in a Capistrano `releases/` directory). On
292
+ Heroku, processes are named after their dyno. Robin's Redis connections
293
+ are named `robin` (see `CLIENT LIST`). The heartbeat warns when Redis
294
+ round trips take over 50ms. A trap the app had already set for one of
295
+ Robin's signals still runs, after Robin's.
296
+
297
+ **systemd.** As a `Type=notify` service, Robin tells systemd when it's
298
+ ready and when it's stopping. With `WatchdogSec=` (a few heartbeat
299
+ intervals, e.g. 30), its heartbeat keeps the watchdog happy.
300
+
301
+ **Signals:**
302
+
303
+ | Signal | Effect |
304
+ |---|---|
305
+ | `TERM` / `INT` | Graceful shutdown: running jobs get `timeout` seconds, then the rest go back to the queue |
306
+ | `TSTP` | Quiet: stop picking up new jobs |
307
+ | `TTIN` (or `INFO`, Ctrl-T, on macOS and BSD) | Log every thread's backtrace |
308
+ | `USR2` | Under `robin swarm`, rolling restart: a replacement child starts before each old one stops |
309
+
310
+ You can also quiet or stop any process from the dashboard.
311
+
312
+ **Resizing.** A running process can change how many workers a capsule
313
+ runs, without a restart: from the dashboard's Busy page (for users with
314
+ the `:process_control` permission), with `POST
315
+ <mount>/api/processes/:identity/resize` and a JSON body like
316
+ `{"concurrency": 20, "capsule": "default"}` (1 to 1000; `capsule`
317
+ defaults to `"default"`), or from Ruby:
318
+
319
+ ```ruby
320
+ Robin::ProcessSet.new.each { |process| process.resize!(20) } # the default capsule
321
+ Robin::ProcessSet.new.first.resize!(2, capsule: "serial")
322
+ ```
323
+
324
+ The process applies it at its next heartbeat. New workers start right
325
+ away; when there are to be fewer, the extra workers finish the job they're
326
+ running (it isn't interrupted) and then stop. Robin grows its Redis
327
+ connection pool to fit more workers, and never shrinks it. If you set
328
+ `pool_size` yourself, it stays as it is, and a resize that would leave
329
+ fewer than two connections beyond the workers is refused with a warning
330
+ in that process's log (as is one sent to a quiet process). Each resize is
331
+ logged and emits a `workers.resized` event, and the capsule's
332
+ `concurrency` reports the new count. It lasts until the process
333
+ restarts, which starts it with its configured concurrency again. There's
334
+ no OS signal for it (`TTIN` already dumps the threads): it travels like
335
+ the dashboard's quiet and stop, as a `RESIZE <count> <capsule>` remote
336
+ command on the process's signal list.
337
+
338
+ An idle worker waits up to `config.fetch_timeout` seconds (default 2) for a
339
+ job before checking whether it should stop, so an idle process stops within
340
+ that time. Lower it for faster shutdowns, at the cost of more round trips
341
+ while idle.
342
+
343
+ `config.fetch_batch` (default 10) caps how many jobs one round trip to
344
+ Redis fetches for a process's waiting workers. Higher means fewer round
345
+ trips under heavy load; `1` fetches one job at a time.
346
+
347
+ **Idle time spent for busy time.** Robin does work while it's quiet so
348
+ that it doesn't have to when jobs come:
349
+
350
+ - A standalone `robin` loads all of Robin and tidies its heap
351
+ (`Process.warmup`) before it starts.
352
+ - If you turn it on, a standalone `robin` collects its garbage when it
353
+ goes idle (see below).
354
+ - On JRuby, a server first runs Robin's push, fetch and middleware code
355
+ for about a second and a half, without Redis or your middleware, so
356
+ the JIT has compiled it before real jobs come (`config.jit_warmup`; on
357
+ by default except on CRuby). Without it, a new process pushes its
358
+ first 20,000 or so jobs about a third as fast as one that warmed up.
359
+ - The cluster's leader wakes its scheduler when the next scheduled job or
360
+ retry it knows of is due, rather than at its next poll (about every
361
+ `config.poll_interval` seconds, 5 by default), so jobs scheduled ahead
362
+ start within milliseconds of their time.
363
+
364
+ **Garbage collection when idle (off by default).** A standalone `robin`
365
+ can run a full GC each time it goes idle: no job running, none finished
366
+ for a second. The next burst of jobs then starts on a clean heap, and
367
+ doesn't pay for the last burst's garbage. Turn it on in `Robin.configure`:
368
+
369
+ ```ruby
370
+ config.idle_gc = true # after any work
371
+ config.idle_gc = 1_000 # only after at least 1,000 jobs since the last one
372
+ ```
373
+
374
+ It has costs, and they're why it's off:
375
+
376
+ - **It stops the process.** A full GC pauses every thread while it runs:
377
+ a few milliseconds for a small heap, tens or hundreds for a large one
378
+ (on JRuby, it pauses the whole JVM). A job that arrives during it waits.
379
+ - **It costs CPU every time the process goes quiet.** For bursty work
380
+ with short gaps, that can mean a full GC every few seconds, each one
381
+ marking the whole heap, where Ruby's own GC would mostly have run
382
+ cheaper minor collections. A minimum number of jobs (`1_000` above)
383
+ limits it to quiet spells that follow real work.
384
+ - **It doesn't make garbage go away.** The work is the same, only moved
385
+ into idle time. If your process is rarely idle, or your jobs allocate
386
+ little, it gains nothing.
387
+
388
+ Try it if your jobs come in bursts after quiet spells, and the first jobs
389
+ of each burst are slower than the rest. Measure the latency of those
390
+ first jobs, and Robin's CPU use, with it on and off. Embedded servers
391
+ never do it, as they share their process with your app.
392
+
393
+ **Health checks.** With `config.health_port = 7433` (or `--health-port`,
394
+ `health_port:` in `robin.yml`, `ROBIN_HEALTH_PORT`), each server answers
395
+ HTTP requests on that port with `200` while its heartbeat reaches Redis
396
+ and it isn't shutting down, and `503` otherwise, plus a JSON report.
397
+ Swarm children listen on the port plus their index. Point a Kubernetes
398
+ probe at it:
399
+
400
+ ```yaml
401
+ livenessProbe:
402
+ httpGet: {path: /, port: 7433}
403
+ # or, without HTTP probes:
404
+ # exec: {command: [bundle, exec, robin, health, --health-port, "7433"]}
405
+ ```
406
+
407
+ The dashboard has its own `/health` (e.g. `/robin/health`) that pings each
408
+ cluster's Redis, for load balancers and uptime monitors.
409
+
410
+ **Swarm** boots your app once and forks `-n` children (default: number of
411
+ CPUs), so memory is shared through copy-on-write. Children that crash are
412
+ restarted. With `config.memory_limit_mb = 1024`, a child that grows past
413
+ the limit restarts gracefully.
414
+
415
+ **Embedded mode** runs a server inside another process:
416
+
417
+ ```ruby
418
+ robin = Robin.configure_embed do |config|
419
+ config.queues = %w[default]
420
+ config.concurrency = 2
421
+ end
422
+ robin.run
423
+ at_exit { robin.stop }
424
+ ```
425
+
426
+ In Puma, use the plugin instead. It runs a server in every worker (or in
427
+ the one process in single mode), started once the app has loaded, so your
428
+ app's own Robin configuration applies, and stopped gracefully whenever
429
+ Puma stops a worker: on shutdown, hot restarts (`SIGUSR2`), phased
430
+ restarts (`SIGUSR1`, `pumactl phased-restart`) and `fork_worker` reforks.
431
+ `robin_config` (optional, after `plugin :robin`) takes the same block as
432
+ `configure_embed` and runs once per worker:
433
+
434
+ ```ruby
435
+ # config/puma.rb
436
+ workers 2
437
+ prune_bundler
438
+ extra_runtime_dependencies ["robin"]
439
+ plugin :robin
440
+ robin_config do |config|
441
+ config.concurrency = 2
442
+ end
443
+ ```
444
+
445
+ Keep `require "robin"` out of `config/puma.rb`, and don't start Robin
446
+ from `before_worker_boot`: both load Robin before the app's bundle is
447
+ set up, which activates Robin's gem dependencies at whatever versions
448
+ are installed, so after a deploy that changes them, phased restarts boot
449
+ workers that fail with "You have already activated ...". The plugin only
450
+ loads Robin in workers, after the app. With `prune_bundler`, add
451
+ `extra_runtime_dependencies ["robin"]` so Puma's master can find the
452
+ plugin without activating the gem (the plugin logs a warning if Robin is
453
+ loaded in a pruned master). Phased restarts need `preload_app!` off,
454
+ which `prune_bundler` implies. Jobs running when a worker stops get
455
+ `config.timeout` to finish before they're pushed back to their queues;
456
+ keep it (plus 3 seconds) under Puma's `worker_shutdown_timeout` (30s by
457
+ default) or Puma kills the worker first (the plugin warns at boot).
458
+
459
+ ## Reliability
460
+
461
+ Robin never loses a job that has been fetched:
462
+
463
+ - **Reliable fetch.** A worker moves a job from its queue into an
464
+ in-flight key of its process in one atomic step. The job is only removed
465
+ from it after it finishes, or after its retry or death has been
466
+ recorded, and it's removed by deleting that key, so a job's arguments
467
+ travel from Redis once, however large. When several workers of a
468
+ process ask for jobs at once, one round trip fetches for all of them (up
469
+ to `config.fetch_batch`, 10 by default), and finished jobs are removed
470
+ with the next round trip, so under load a job costs a fraction of a
471
+ round trip. Jobs fetched for a worker stay in flight until they're done;
472
+ ones no worker takes within a second (because every worker got busy) go
473
+ back to their queue, as do the fetched jobs of a process that stops or
474
+ a queue that's paused. A finished job is removed with the next round
475
+ trip of its process, or by a thread of its own if none comes within 2
476
+ milliseconds, so a crash can make only jobs that finished just before
477
+ it run again.
478
+ - **Jobs must be idempotent.** Delivery is at least once: a job runs
479
+ again if its process crashes while it runs, or just after it finishes,
480
+ and if a shutdown interrupts it. Write jobs so that running one twice
481
+ does no harm (check whether the work was already done, or use unique
482
+ keys or upserts).
483
+ - **Recovery.** If a process crashes, is OOM-killed or loses its node, its
484
+ heartbeat expires. The cluster leader then moves that process's in-flight
485
+ jobs back to the front of their queues. `config.recovery_handlers` (like
486
+ `death_handlers`) are called with each recovered job and whether it was
487
+ a poison pill. A server with `config.leader_candidate = false` never
488
+ becomes the leader, so it never runs the leader's duties (periodic jobs,
489
+ recovery).
490
+ - **Poison pills.** A job that was running when its process died more than
491
+ `config.max_recoveries` (3) times goes to the dead set instead of taking
492
+ down process after process.
493
+ - **Graceful shutdown.** Jobs still running when the shutdown timeout ends
494
+ are interrupted and pushed back to their queue.
495
+ - **Reliable push.** With `config.reliable_push = true`, pushes made while
496
+ Redis is unreachable are buffered in memory (up to 1,000 by default) and
497
+ sent in order once it's back. When the process exits it tries once more;
498
+ jobs it still can't push, and jobs dropped from a full buffer, are
499
+ reported to the error handlers (as `Robin::ReliablePush::Lost` and
500
+ `Dropped`).
501
+ - **Deferred push.** `config.push_deferrer` can hold pushed jobs back, e.g.
502
+ until a database transaction commits: it's called with the jobs and a
503
+ block that sends them, and returns truthy when it took over. The Rails
504
+ integration uses it for [transaction-aware pushes](#rails-and-activejob).
505
+
506
+ Pause a queue (workers skip it until it's unpaused) with
507
+ `Robin::Queue.new("mailers").pause!` or from the dashboard.
508
+
509
+ Pause a job class across every queue with
510
+ `Robin::JobClass.new(ReportJob).pause!`, or from the class's page in the
511
+ dashboard (Robin-only), when one kind of job is misbehaving but its queues
512
+ hold others that should keep running. As workers fetch its jobs they set
513
+ them aside instead of running them; `unpause!` puts them back on the front
514
+ of their queues, oldest first, and returns how many. Every process sees a
515
+ pause within a heartbeat (`heartbeat_interval`, 5 seconds by default), and
516
+ jobs already running finish. An Active Job job is paused by the class it
517
+ wraps. `Robin::JobClass.list` gives each paused class and how many of its
518
+ jobs are set aside, and the dashboard's Queues page lists them too.
519
+
520
+ Shutdowns and failovers are handled conservatively too:
521
+
522
+ - Jobs pushed back by a graceful shutdown don't count toward the
523
+ poison-pill limit, even if the process is killed while stopping.
524
+ - `Robin::Shutdown` only interrupts a job's `perform`, never Robin's own
525
+ bookkeeping, and interrupted jobs share one 3-second grace period.
526
+ - After a Redis failover (READONLY, LOADING and similar replies) the
527
+ connection is dropped so the next command reaches the new primary. A
528
+ command is only resent if none of it reached the server.
529
+ - The heartbeat and the leader's duties use their own Redis connections
530
+ and higher-priority threads, so busy jobs can't starve them. If a
531
+ heartbeat lapses anyway, the process logs a warning: its running jobs may
532
+ have been recovered and run elsewhere.
533
+ - Every minute a process checks its own in-flight keys and puts back jobs
534
+ that no worker holds (for example, a worker that died after finishing a
535
+ job but before acknowledging it, or a round trip whose reply never
536
+ arrived).
537
+
538
+ At boot, Robin refuses to start on Redis older than 6.2, with a concurrency
539
+ or shutdown timeout of 0, a Redis pool smaller than the worker count plus
540
+ two, or a queue listed twice. It warns if Redis may evict keys
541
+ (`maxmemory-policy` other than `noeviction`). An exception in a `:startup`
542
+ hook stops the boot.
543
+
544
+ ## Batches and workflows
545
+
546
+ ```ruby
547
+ batch = Robin::Batch.new
548
+ batch.description = "Import #{file.name}"
549
+ batch.on(:success, ImportCallbacks, "file_id" => file.id) # every job succeeded
550
+ batch.on(:complete, "ImportCallbacks#finished", "file_id" => file.id) # every job ran
551
+ batch.on(:death, ImportCallbacks) # a job died
552
+ bid = batch.jobs do
553
+ rows.each_slice(100) { |slice| ImportRows.perform_async(file.id, slice) }
554
+ end
555
+
556
+ class ImportCallbacks
557
+ def on_success(status, options)
558
+ Notifier.import_done(options["file_id"], status.total)
559
+ end
560
+
561
+ def finished(status, options) = ...
562
+ end
563
+ ```
564
+
565
+ Inside a batch job, `batch` returns the job's batch, so a job can add more
566
+ jobs to its own batch (`batch.jobs { ... }`). A batch created inside
567
+ another batch's `jobs` block, or inside a running batch job, becomes a
568
+ child, and the parent isn't complete until its children are. You can build
569
+ multi-step workflows by nesting batches and chaining callbacks.
570
+
571
+ `batch.jobs(atomic: true) { ... }` pushes the block's jobs when it
572
+ finishes, all at once, and none at all if it raises (a batch it created is
573
+ deleted again). Without `atomic`, jobs are pushed as the block pushes
574
+ them.
575
+
576
+ Inspect batches with `Robin::Batch::Status.new(bid)` or on the dashboard.
577
+
578
+ To show a batch's progress on your own pages (a progress bar for an
579
+ import, say), mount `Robin::Web::BatchStatus` and poll it from the browser:
580
+
581
+ ```ruby
582
+ # config/routes.rb
583
+ mount Robin::Web::BatchStatus.new => "/batch_status"
584
+ ```
585
+
586
+ `GET /batch_status/<bid>.json` returns `{"bid", "total", "pending",
587
+ "failures", "complete", "success", "dead", "cancelled"}`, or a 404 once the
588
+ batch is gone. It skips the dashboard's authorization, so it shows counts
589
+ and states only (never jobs, arguments, errors or the description), and a
590
+ bid's 80 random bits are what keep other people's batches out of reach.
591
+ Pass `config:` to read another Redis.
592
+
593
+ Tag a batch to find it later. `Robin::Batch.list(tag: "imports")` lists
594
+ the batches with that tag, newest first, and so does the dashboard's
595
+ Batches page when you click a tag:
596
+
597
+ ```ruby
598
+ batch.tags = ["imports", "customer:#{file.customer_id}"] # before the first `jobs`
599
+ ```
600
+
601
+ Jobs are pushed as they're created inside `jobs`, never collected first,
602
+ so a batch of any size takes no more memory in the process creating it.
603
+
604
+ Cancel a batch with `Robin::Batch.new(bid).cancel!` (or from the
605
+ dashboard). Its child batches are cancelled too. Their jobs that haven't
606
+ started are skipped, and running ones see `cancelled?` turn true, so they
607
+ can stop early. A cancelled batch still completes once every job has run
608
+ or been skipped, so `complete` callbacks fire (check
609
+ `status.cancelled?`), but `success` never does, and a cancelled child
610
+ counts as a failure in its parent.
611
+
612
+ ## Rate limiting
613
+
614
+ Limiters work from any Ruby code. If a job hits a limit, it is rescheduled
615
+ with backoff rather than counted as a failure.
616
+
617
+ ```ruby
618
+ STRIPE = Robin::Limiter.concurrent("stripe", 10, wait_timeout: 5, lock_timeout: 30)
619
+ GEO = Robin::Limiter.bucket("geocoder", 50, :second)
620
+ EMAIL = Robin::Limiter.window("smtp", 1_000, :hour) # sliding window
621
+ CRAWL = Robin::Limiter.leaky("crawler", 60, 60) # drips 1/second
622
+ OPENAI = Robin::Limiter.points("llm-tokens", 90_000, 60) # token bucket
623
+
624
+ def perform(user_id)
625
+ STRIPE.within_limit { Stripe::Charge.create(...) }
626
+ OPENAI.within_limit(points: estimated_tokens) { ... }
627
+ end
628
+ ```
629
+
630
+ When a limiter doesn't free up within `wait_timeout`, it raises
631
+ `Robin::Limiter::OverLimit`. Inside a job, Robin catches that and
632
+ reschedules the job with a jittered backoff. After
633
+ `config.max_rate_limit_retries` (20) reschedules, normal retries apply.
634
+ Points limiters can `adjust` or `refund` points once the real cost is known.
635
+
636
+ For work that can simply be dropped, such as refreshing a cache, give the
637
+ limiter `policy: :skip`: when it doesn't free up in time, `within_limit`
638
+ returns nil without running the block (and the dashboard still counts it
639
+ as over the limit).
640
+
641
+ ```ruby
642
+ REFRESH = Robin::Limiter.bucket("price-refresh", 10, :minute, wait_timeout: 0, policy: :skip)
643
+ REFRESH.within_limit { PriceCache.refresh(sku) }
644
+ ```
645
+
646
+ Bucket and window limiters can count one call as several uses, e.g. an
647
+ API call that covers a batch of records: `GEO.within_limit(used: 10) { ... }`.
648
+
649
+ Limiters keep their state in the configured Redis. To keep them in another
650
+ Redis (a shard of their own, say), wrap the calls in `Robin::Limiter.via`:
651
+
652
+ ```ruby
653
+ LIMITS = Robin::Redis::Pool.new(size: 10) { Robin::Redis::Connection.new(url: ENV["LIMITER_REDIS_URL"]) }
654
+ Robin::Limiter.via(LIMITS) { STRIPE.within_limit { charge! } }
655
+ ```
656
+
657
+ The dashboard's Limiters page shows the limiters in the configured Redis
658
+ only.
659
+
660
+ Every kind of limiter also takes `reschedule:` (how many times a job that
661
+ hits it is rescheduled before it counts as a failure; 0 fails at once) and
662
+ `backoff:`, a callable that picks the delay:
663
+
664
+ ```ruby
665
+ PARTNER = Robin::Limiter.window("partner-api", 100, :minute,
666
+ reschedule: 5, backoff: ->(count, error, payload) { error.retry_after || 30 * count })
667
+
668
+ Robin.configure_server do |config|
669
+ config.rate_limit_backoff = ->(count, error, payload) { 10 * count } # for every limiter without one
670
+ end
671
+ ```
672
+
673
+ A backoff returning nil gets the default (5 seconds times the count, or
674
+ however long the limiter says capacity takes to free up, up to 5 minutes,
675
+ with jitter). `Robin::Limiter.unlimited` never limits and never touches
676
+ Redis: swap it in where a limit shouldn't apply, such as in tests.
677
+
678
+ ## Periodic (cron) jobs
679
+
680
+ ```ruby
681
+ Robin.configure_server do |config|
682
+ config.periodic do |cron|
683
+ cron.register "*/5 * * * *", "PruneSessions"
684
+ cron.register "0 9 * * MON-FRI", "DailyDigest", queue: "mailers", tz: "America/New_York"
685
+ cron.register "@hourly", "SyncInventory", args: ["warehouse-1"], description: "Pull stock levels"
686
+ end
687
+ end
688
+ ```
689
+
690
+ Only the elected leader enqueues periodic jobs, and each tick is
691
+ deduplicated in Redis, so a job runs once per tick no matter how many
692
+ processes you run. After a leader failover, at most one missed tick is
693
+ caught up. The dashboard lists every periodic job with its next run and
694
+ history, and lets you run one now or pause it.
695
+
696
+ IANA time zones need the `tzinfo` gem. `UTC` and fixed offsets like `+05:30`
697
+ work without it.
698
+
699
+ In a Rails app, a periodic job can be an ActiveJob class: Robin pushes it
700
+ like `perform_later` would (on the given queue, else the job's own), but
701
+ without running ActiveJob's enqueue callbacks.
702
+
703
+ To catch a broken schedule before it ships, check it in your test suite.
704
+ `Robin::Periodic.verify!` takes the block you give `config.periodic` (or a
705
+ config, to check all of its blocks) and raises listing every bad cron
706
+ expression or time zone, class that doesn't exist or isn't a job, and
707
+ argument that JSON would change:
708
+
709
+ ```ruby
710
+ # config/initializers/robin.rb
711
+ SCHEDULE = ->(cron) { cron.register "0 3 * * *", "NightlyReport" }
712
+ Robin.configure_server { |config| config.periodic(&SCHEDULE) }
713
+
714
+ # test/robin_schedule_test.rb
715
+ test("the schedule is valid") { Robin::Periodic.verify!(&SCHEDULE) }
716
+ ```
717
+
718
+ ## Unique, debounced, expiring and encrypted jobs
719
+
720
+ ```ruby
721
+ class RebuildSearchIndex
722
+ include Robin::Job
723
+ robin_options unique_for: 600 # duplicate pushes (same args) for 10 min are dropped
724
+ end
725
+
726
+ class SyncContact
727
+ include Robin::Job
728
+ robin_options debounce: 30 # runs once, 30s after the *last* push
729
+ end
730
+
731
+ class SendFlashSale
732
+ include Robin::Job
733
+ robin_options expires_in: 3600 # skipped if it hasn't started within an hour
734
+ end
735
+
736
+ class ChargeCard
737
+ include Robin::Job
738
+ robin_options encrypt: true # arguments are encrypted at rest
739
+ end
740
+
741
+ Robin.configure do |config|
742
+ config.encryption_keys = {1 => ENV["ROBIN_KEY_V1"], 2 => ENV["ROBIN_KEY_V2"]}
743
+ end
744
+ ```
745
+
746
+ - **Unique locks** are released when the job succeeds (the default) or, with
747
+ `unique_until: :start`, when it starts. Define `robin_unique_args(args)` on
748
+ the class to choose which arguments count toward uniqueness.
749
+ - **Encryption** uses AES-256-GCM, with the job id as authenticated data.
750
+ New jobs are encrypted with the highest-numbered key, and any listed key
751
+ can decrypt, so you can rotate keys without downtime. Decrypted arguments
752
+ are never written back to Redis, including in retries and the dead set.
753
+
754
+ ## Long-running (iterable) jobs
755
+
756
+ Iterable jobs save their position as they go. On deploys and restarts they
757
+ stop between iterations and resume elsewhere, without redoing work.
758
+
759
+ ```ruby
760
+ class BackfillAvatars
761
+ include Robin::IterableJob
762
+
763
+ def build_enumerator(account_id, cursor:)
764
+ active_record_records_enumerator(User.where(account_id: account_id), cursor: cursor)
765
+ end
766
+
767
+ def each_iteration(user, _account_id)
768
+ user.regenerate_avatar!
769
+ end
770
+ end
771
+ ```
772
+
773
+ Also available: `array_enumerator`, `active_record_batches_enumerator` and
774
+ `csv_enumerator`, plus `on_start`, `on_resume`, `on_cancel`, `on_stop`
775
+ (after an interruption or a cancellation) and `on_complete` hooks, and
776
+ `around_iteration`, which wraps each iteration (yield to run it). Inside
777
+ them and `each_iteration`, `cursor` is the last finished item's cursor and
778
+ `current_object` the item being worked on.
779
+
780
+ A long iteration can hold a worker for hours. Set
781
+ `config.max_iteration_runtime` (seconds), or the `max_iteration_runtime`
782
+ option on a job, and a run that goes past it checkpoints and goes back on
783
+ its queue, as it does on a deploy, so other jobs get a turn.
784
+
785
+ If an iteration raises, the progress made so far is saved first, so the
786
+ retry resumes after the last finished item. A cancelled iterable job
787
+ stops at its next checkpoint, about every five seconds: saving its
788
+ position, reporting progress and checking for cancellation share one
789
+ Redis round trip. Progress is a percentage when the enumerator knows its
790
+ size (`array_enumerator` does), and otherwise a count of the items done
791
+ (the Active Record and CSV enumerators). Iterable jobs don't define
792
+ `perform` (Robin raises if one does). Robin logs a warning when a single
793
+ iteration takes longer than the shutdown timeout, since a deploy could
794
+ interrupt it midway.
795
+
796
+ ## Progress and cancellation
797
+
798
+ ```ruby
799
+ def perform(ids)
800
+ ids.each_with_index do |id, i|
801
+ return if cancelled?
802
+ process(id)
803
+ progress!((i + 1) * 100.0 / ids.size, "#{i + 1} of #{ids.size}")
804
+ end
805
+ end
806
+
807
+ jid = Export.perform_async(ids)
808
+ Robin.cancel!(jid) # jobs that haven't started are skipped; running ones see cancelled?
809
+ Robin.cancel!(*jids) # several, in one round trip
810
+ ```
811
+
812
+ Progress shows live on the dashboard's Busy page. Pass `nil` for the
813
+ percentage when the job can't tell how far along it is: the page shows the
814
+ message without a percentage.
815
+
816
+ ## Middleware
817
+
818
+ ```ruby
819
+ class TenantMiddleware
820
+ include Robin::Middleware::ConfigAware # optional: gives you config, redis, logger
821
+
822
+ # client side: return without yielding to stop the push
823
+ def call(job_class, payload, queue, redis)
824
+ payload["tenant"] ||= Current.tenant_id
825
+ yield
826
+ end
827
+ end
828
+
829
+ class TenantServerMiddleware
830
+ def call(job, payload, queue)
831
+ Current.set(tenant_id: payload["tenant"]) { yield }
832
+ end
833
+ end
834
+
835
+ Robin.configure do |config|
836
+ config.client_middleware { |chain| chain.add TenantMiddleware }
837
+ end
838
+ Robin.configure_server do |config|
839
+ config.server_middleware { |chain| chain.add TenantServerMiddleware }
840
+ end
841
+ ```
842
+
843
+ A middleware is instantiated for every job. If yours keeps no per-job state
844
+ in instance variables, `include Robin::Middleware::Stateless` and one
845
+ instance serves every job instead (on every thread). The built-in
846
+ middleware all work this way.
847
+
848
+ A middleware that only matters to some jobs can say which with a class
849
+ method `applies?`, taking the same arguments as `call`. The chain skips it
850
+ for the others, without building it:
851
+
852
+ ```ruby
853
+ class AuditMiddleware
854
+ def self.applies?(_job, payload, _queue) = payload.key?("audit")
855
+
856
+ def call(job, payload, queue) = Audit.around(payload) { yield }
857
+ end
858
+ ```
859
+
860
+ The built-in middleware (encryption, expiry, uniqueness, debouncing,
861
+ batches, profiling) use it, so jobs that use none of those features skip
862
+ them.
863
+
864
+ ## Errors, retries and the dead set
865
+
866
+ Failed jobs are retried with exponential backoff. The delay starts at about
867
+ 15 seconds, grows 1.7x per attempt with ±20% jitter, and is capped at one
868
+ week. The default 20 retries span about ten days. After the last retry the
869
+ job moves to the dead set, which keeps 10,000 jobs for 180 days by default.
870
+ From the dashboard you can retry dead jobs, with or without editing their
871
+ arguments first.
872
+
873
+ ```ruby
874
+ class CallWebhook
875
+ include Robin::Job
876
+ robin_options retry: 8
877
+
878
+ robin_retry_in do |count, exception, payload|
879
+ case exception
880
+ when Net::HTTPTooManyRequests then 60 * count
881
+ when ArgumentError then :kill # straight to the dead set
882
+ when RecordGone then :discard # drop it
883
+ end # nil: default backoff
884
+ end
885
+
886
+ robin_retries_exhausted do |payload, exception|
887
+ Alerts.webhook_gave_up(payload["args"].first, exception)
888
+ :discard if payload["args"].first.nil? # drop it instead of keeping it in the dead set
889
+ end
890
+ end
891
+
892
+ Robin.configure do |config|
893
+ config.error_handlers << ->(ex, context, _config) { ErrorTracker.notify(ex, context) }
894
+ config.death_handlers << ->(payload, ex) { ... }
895
+ end
896
+ ```
897
+
898
+ Error handlers run in the failed job's logging context (its class, jid,
899
+ batch and tags). The default handler logs each error as one entry: the
900
+ error, its context and the start of its backtrace.
901
+
902
+ Things that aren't errors but are worth knowing about (a job expired,
903
+ skipped or rate limited, a worker died, Redis is slow, the heartbeat
904
+ lapsed, a reliable push was buffered or lost, and so on) are events.
905
+ `config.event_handlers` are called with each event's name and a Hash
906
+ about it; `Robin::Events` lists them. In a Rails app they're published through
907
+ `ActiveSupport::Notifications` as `"<event>.robin"`.
908
+
909
+ ```ruby
910
+ config.event_handlers << ->(name, payload) { Metrics.increment("robin.#{name}") }
911
+ ```
912
+
913
+ `retry_for:` limits retries by time instead: the job keeps retrying (with
914
+ the same backoff) until a retry would come more than that many seconds
915
+ after its first failure, and then dies. Without an explicit `retry:` count
916
+ it can take as many attempts as fit; with one, whichever limit comes first
917
+ applies. `retry_queue:` sends a job's retries to another queue, e.g. a
918
+ low-priority one, so failing jobs don't hold up fresh ones.
919
+
920
+ Delays returned by `robin_retry_in` get up to 10% (at most 10 seconds) of
921
+ jitter, so jobs that failed together don't all retry at the same moment.
922
+ `config.retry_backoff` replaces the backoff altogether: a callable
923
+ `(failures, exception, payload, custom)` returning the seconds before the
924
+ retry, where `custom` is the job's `robin_retry_in` answer (or nil); Robin
925
+ adds no jitter to it. An ActiveJob job's retry hooks come from the class
926
+ it wraps.
927
+
928
+ Death handlers run whenever a job won't run again: when it moves to the
929
+ dead set, and when it's discarded (`retry: false`, `:discard` from
930
+ `robin_retry_in`, `dead: false`, or `:discard` from `robin_retries_exhausted`).
931
+ Discarded jobs have `payload["discarded_at"]` set.
932
+
933
+ A retry or dead job is stored as it was pushed, plus the failure details.
934
+ If `perform` or a middleware changed the job's arguments in memory, those
935
+ changes aren't saved.
936
+
937
+ Jobs with `backtrace: true` (or a number of lines) keep the error's
938
+ backtrace, compressed. `config.backtrace_cleaner` trims it first (in a Rails
939
+ app, `Rails.backtrace_cleaner`); read it back with
940
+ `entry.error_backtrace`. `retry_all`, `kill_all` and the dashboard's
941
+ "all" actions work through a set a page at a time, so huge sets never have
942
+ to fit in memory.
943
+
944
+ ## Metrics
945
+
946
+ Each process tracks per-class counts, total time and a latency histogram in
947
+ memory (times and histograms cover successful runs only), and flushes them with its heartbeat, so recording adds no Redis
948
+ round-trips per job. Data kept:
949
+
950
+ - per-minute data for the last 8 hours;
951
+ - hourly rollups for `config.metrics_retention_days` (90 by default);
952
+ - queue size and latency history sampled by the leader.
953
+
954
+ ```ruby
955
+ Robin.configure do |config|
956
+ config.statsd = Robin::Statsd.new(host: "127.0.0.1", port: 8125, prefix: "robin",
957
+ tags: {env: "production"}, dogstatsd: true)
958
+ end
959
+ ```
960
+
961
+ Besides job counts and timings and queue sizes and latency, StatsD gets
962
+ `batch.created`, `batch.complete`, `batch.success`, `batch.death` and
963
+ `batch.cancelled` counters and a `batch.duration` timing (creation to
964
+ completion), plus `jobs.recovered` (in-flight jobs put back from dead
965
+ processes), `jobs.orphaned` (in-flight jobs no worker held) and
966
+ `jobs.poison` (by class) counters. Under DogStatsD, the batch metrics
967
+ carry the batch's tags (`tag:<tag>` for each).
968
+
969
+ For percentiles across every process, add `distributions: true` (DogStatsD
970
+ only) to also send Datadog distributions: `jobs.perform_dist` (by class;
971
+ this sends a packet per successful job) and `batch.duration_dist`.
972
+
973
+ The dashboard also serves Prometheus metrics at `<mount>/metrics`. Browsers
974
+ asking for HTML get the Metrics page at that path instead.
975
+
976
+ **Deploy markers.** Mark each deploy, and the dashboard's metrics charts
977
+ draw a line where it happened, so a change in throughput, failures or
978
+ timing can be matched to the deploy that caused it:
979
+
980
+ ```ruby
981
+ Robin::Deploy.mark!("v42 #{`git rev-parse --short HEAD`.strip}")
982
+ ```
983
+
984
+ or `bundle exec robin deploy "v42"` from a deploy script. Every process
985
+ may mark the same deploy (at boot, say): a label already marked in the
986
+ last hour isn't marked again. Markers are kept as long as hourly metrics.
987
+
988
+ **GVL contention.** Only one of a process's threads runs Ruby code at a
989
+ time; the others wait for the GVL. With the C extension, each process
990
+ measures how long its threads wait, and every heartbeat records its
991
+ contention: the share of the time its threads could have spent waiting
992
+ that they did. Near 0%, threads rarely get in
993
+ each other's way; near 100%, every thread wanted to run all the time, so
994
+ more threads won't run more Ruby code. Threads waiting on I/O (Redis, a
995
+ database, an HTTP call) don't wait for the GVL, so an I/O-heavy process
996
+ stays low however many threads it has.
997
+
998
+ The dashboard shows each process's contention, StatsD gets it as the
999
+ `gvl.contention` gauge, Prometheus as
1000
+ `robin_process_gvl_contention_percent`, and a process whose contention stays at 50% or
1001
+ more for three heartbeats in a row logs a warning (at most every 10
1002
+ minutes) and emits a `gvl.contended` event. When it's persistently high,
1003
+ CPU-heavy jobs would finish sooner with fewer threads per process and
1004
+ more processes (or `robin swarm`). Robin doesn't change your concurrency
1005
+ itself. Measuring costs under 2% even in code that does nothing but
1006
+ switch threads; `config.gvl_monitoring = false` turns it off. JRuby has
1007
+ no GVL, so there's nothing to measure (nor is there on a 32-bit platform,
1008
+ where the extension can't keep a wait's start with its thread).
1009
+
1010
+ ## Tracing and profiling
1011
+
1012
+ **OpenTelemetry.** With the `opentelemetry-api` gem (and an SDK exporting
1013
+ somewhere), Robin traces pushes and runs:
1014
+
1015
+ ```ruby
1016
+ require "robin/opentelemetry"
1017
+
1018
+ Robin.configure do |config|
1019
+ Robin::OpenTelemetry.install(config) # runs are children of their push
1020
+ # Robin::OpenTelemetry.install(config, parent: :link) # or new traces, linked to it
1021
+ end
1022
+ ```
1023
+
1024
+ Pushing opens a `publish <queue>` producer span and stores the trace
1025
+ context in the job; running it opens a `process <queue>` consumer span.
1026
+ Both follow the messaging semantic conventions, plus `robin.job.class`
1027
+ and `robin.job.retry_count`.
1028
+
1029
+ **Profiling one job.** With `gem "vernier"` (CRuby only), push a job with
1030
+ `profile:` to record a sampling profile of that run, even if it fails:
1031
+
1032
+ ```ruby
1033
+ ReportJob.set(profile: true).perform_async(42)
1034
+ ReportJob.set(profile: "big export").perform_async(43) # labelled
1035
+ ```
1036
+
1037
+ Profiles are kept in Redis for a week (the newest 100). The dashboard lists
1038
+ them and opens them in the [Firefox Profiler](https://profiler.firefox.com);
1039
+ from Ruby, use `Robin::Profiler.list` and `Robin::Profiler.data(jid)`. A
1040
+ process profiles one job at a time; others asking meanwhile run unprofiled.
1041
+
1042
+ ## Web dashboard
1043
+
1044
+ ```ruby
1045
+ # config/routes.rb
1046
+ require "robin/web"
1047
+ mount Robin::Web => "/robin"
1048
+
1049
+ # or config.ru
1050
+ require "robin/web"
1051
+ run Robin::Web
1052
+ ```
1053
+
1054
+ The dashboard is a TypeScript single-page app built with bun and shipped
1055
+ pre-built inside the gem. Its pages are the dashboard, busy processes and
1056
+ jobs (with live progress), queues, scheduled, retries, dead, batches,
1057
+ limiters, cron, job profiles, and per-class metrics. It updates live by polling every
1058
+ few seconds. Set `Robin::Web.live_updates = :stream` to push updates
1059
+ over Server-Sent Events instead. Each open stream occupies a web server
1060
+ thread, so streaming is off by default.
1061
+
1062
+ ```ruby
1063
+ Robin::Web.authorize do |env, action, resource|
1064
+ user = env["warden"]&.user
1065
+ action == :read ? user&.staff? : user&.admin?
1066
+ end
1067
+ Robin::Web.read_only = true # nobody can change anything
1068
+ Robin::Web.clusters = {"us-east" => us_config, "eu-west" => eu_config} # multi-cluster
1069
+ ```
1070
+
1071
+ **Rows of your own on job pages.** Add a row to every job's details, as
1072
+ text or as a link (http(s) URLs and paths only). The block gets the job's
1073
+ payload, and returning nil leaves the row out:
1074
+
1075
+ ```ruby
1076
+ Robin::Web.job_info("Customer") do |job|
1077
+ id = job["args"].first
1078
+ {text: "##{id}", href: "/admin/customers/#{id}"} if job["class"] == "InvoiceJob"
1079
+ end
1080
+ ```
1081
+
1082
+ **Assets from a CDN.** Serve the dashboard's scripts, stylesheets, fonts
1083
+ and images from elsewhere by copying `lib/robin/web/dist/assets` (that is,
1084
+ `Robin::Web::DIST + "/assets"`) there for each Robin version. The CDN must
1085
+ send CORS headers, because the scripts are modules. The content security
1086
+ policy lets the dashboard load from that host.
1087
+
1088
+ ```ruby
1089
+ Robin::Web.assets_path = "https://cdn.example.com/robin/#{Robin::VERSION}/"
1090
+ ```
1091
+
1092
+ **Languages.** The dashboard speaks Arabic, Chinese (simplified and
1093
+ traditional), Czech, Danish, Dutch, English, French, German, Hebrew,
1094
+ Hindi, Italian, Japanese, Korean, Norwegian, Persian, Polish, Portuguese
1095
+ (Brazil), Russian, Spanish, Swedish, Turkish, Ukrainian, Urdu and
1096
+ Vietnamese, with right-to-left layouts for Arabic, Hebrew, Persian and
1097
+ Urdu. It picks the first of the browser's languages it knows, and can be
1098
+ switched under Settings (the choice is remembered). Numbers, dates and
1099
+ durations follow the language too. Each language is a separate file,
1100
+ loaded only when it's chosen. The translations were machine-made; native
1101
+ speakers are welcome to improve them (see [Development](#development)).
1102
+
1103
+ Changes require a same-origin request with the `X-Robin-Request` header
1104
+ (and, from browsers that send it, a same-origin `Sec-Fetch-Site`), so
1105
+ cross-site requests can't make them (CSRF protection). The dashboard
1106
+ also sends a strict Content Security Policy.
1107
+
1108
+ **Your own pages.** Add a tab to the dashboard without writing any
1109
+ frontend code: describe the page with blocks, and attach actions to its
1110
+ buttons.
1111
+
1112
+ ```ruby
1113
+ class BillingPage < Robin::Web::Extension
1114
+ slug "billing"
1115
+ title "Billing"
1116
+
1117
+ def blocks(request) # request.config, request.params, request.env
1118
+ [
1119
+ {type: "stats", items: [{label: "Invoices waiting", value: Robin::Queue.new("invoices").size}]},
1120
+ {type: "heading", text: "Overdue"},
1121
+ {type: "table", columns: ["Customer", "Total"], rows: Invoice.overdue.map { [_1.customer, _1.total] }},
1122
+ {type: "text", text: "Invoices go out at 6am."},
1123
+ {type: "actions", items: [{label: "Send now", action: "send", confirm: "Send every invoice?"}]}
1124
+ ]
1125
+ end
1126
+
1127
+ action "send" do |request|
1128
+ SendInvoices.perform_async
1129
+ "Sending invoices" # the message the user sees
1130
+ end
1131
+ end
1132
+
1133
+ Robin::Web.register(BillingPage)
1134
+ ```
1135
+
1136
+ Values in blocks are always shown as text, never as HTML. Running an
1137
+ action needs the `:extensions` permission (the resource passed to
1138
+ `authorize` is `"extension:billing/send"`).
1139
+
1140
+ For anything blocks can't show, a page can bring its own frontend code: a
1141
+ JavaScript module (and a stylesheet), served from a directory of assets,
1142
+ plus JSON data endpoints for it to call.
1143
+
1144
+ ```ruby
1145
+ class ReportsPage < Robin::Web::Extension
1146
+ slug "reports"
1147
+ assets File.expand_path("reports", __dir__) # served at <mount>/extensions/reports/
1148
+ script "reports.js"
1149
+ stylesheet "reports.css"
1150
+ translations File.expand_path("reports/locales", __dir__) # de.yml, ja.yml, ...
1151
+
1152
+ data "late" do |request| # GET api/extensions/reports/data/late
1153
+ Report.late.limit(Integer(request.params.fetch("limit", 20))).map(&:to_h)
1154
+ end
1155
+
1156
+ action "rerun" do |request| # request.body is the JSON sent
1157
+ RerunReport.perform_async(request.body.fetch("id"))
1158
+ request.t("Rerunning {name}", name: request.body.fetch("name"))
1159
+ end
1160
+ end
1161
+ ```
1162
+
1163
+ ```js
1164
+ // reports/reports.js: called with an element to render into and a context
1165
+ export default function render(element, robin) {
1166
+ const heading = document.createElement("h2");
1167
+ heading.textContent = robin.t("Late reports");
1168
+ element.append(heading);
1169
+ robin.data("late", { limit: 50 }).then((reports) => { /* draw them */ });
1170
+ // robin.action("rerun", { id, name }) resolves to the action's message;
1171
+ // robin.toast(message), robin.navigate("/queues"), robin.locale, robin.slug
1172
+ return () => { /* the page went away: stop timers, remove listeners */ };
1173
+ }
1174
+ ```
1175
+
1176
+ The script runs with the dashboard's own Content Security Policy, so it
1177
+ can load further files from its assets directory but nothing from other
1178
+ sites, and inline `<script>` and `<style>` elements are blocked. Translations are YAML files
1179
+ mapping English text to the language (`de.yml`: `Late reports: Verspätete
1180
+ Berichte`); they translate the page's title in the navigation, text passed
1181
+ to `request.t` in blocks and actions, and `robin.t` in the script. Viewing
1182
+ the page, its assets and its data needs the `:read` permission.
1183
+
1184
+ ## Rails and ActiveJob
1185
+
1186
+ The Rails integration is part of the `robin` gem, and turns itself on in
1187
+ a Rails app: Bundler loads robin after Rails, and robin then joins the
1188
+ app's boot. Elsewhere it stays off and loads nothing of Rails. If robin is
1189
+ required before Rails (or outside Bundler.require), add
1190
+ `require "robin/rails"` after Rails. It needs Rails (railties and Active
1191
+ Job) 7.2 or newer, and is tested in CI against the latest 7.2, 8.0 and 8.1
1192
+ releases.
1193
+
1194
+ ```ruby
1195
+ # Gemfile
1196
+ gem "robin"
1197
+ ```
1198
+
1199
+ ```sh
1200
+ bin/rails generate robin:install # config/robin.yml, an initializer, the dashboard route
1201
+ bin/rails generate robin:job report queue:low retry:5 # app/jobs/report_job.rb and its test
1202
+ bundle exec robin # boots the Rails app in this directory
1203
+ ```
1204
+
1205
+ `robin:install` creates:
1206
+
1207
+ - `config/robin.yml`, the server settings (concurrency, queues, timeout,
1208
+ per-environment sections) read by `bundle exec robin`
1209
+ - `config/initializers/robin.rb`, with `Robin.configure` (every process)
1210
+ and `Robin.configure_server` (only the robin process) blocks, and a
1211
+ commented `Robin::Web.authorize` example
1212
+ - a `mount Robin::Web => "/robin"` line in `config/routes.rb`. The
1213
+ dashboard can see and change every job, so protect it: with
1214
+ `Robin::Web.authorize`, or by mounting it inside an authenticated route
1215
+ constraint (e.g. Devise's `authenticate :user, ->(user) { user.admin? } do ... end`).
1216
+
1217
+ Running it again changes nothing.
1218
+
1219
+ `robin:job NAME [option:value ...]` creates `app/jobs/NAME_job.rb`, a
1220
+ `Robin::Job` with those `robin_options` (`queue` defaults to `"default"`;
1221
+ numbers, `true` and `false` stay literals, and a bare `option` means
1222
+ `true`), and a test: `spec/jobs/NAME_job_spec.rb` when the app generates
1223
+ RSpec tests, `test/jobs/NAME_job_test.rb` otherwise. Pass
1224
+ `--test-framework=rspec` or `test_unit` to choose.
1225
+
1226
+ In a Rails app:
1227
+
1228
+ - **Executor and reloader.** Every job runs inside the app's reloader, so
1229
+ code reloading, query caches and connection checkin work as in a request.
1230
+ - **`bundle exec robin`** boots the Rails app in the current directory (or
1231
+ the one given with `-r`), and `RAILS_ENV` picks the `config/robin.yml`
1232
+ section. `-e production` sets `RAILS_ENV` too.
1233
+ - **Environment and logging.** `Robin.config.environment` follows
1234
+ `Rails.env` (set before your initializers, so they can override it).
1235
+ Outside the robin process (web, console), Robin logs to `Rails.logger`.
1236
+ - **Error reporting.** Errors go to the Rails error reporter
1237
+ (`Rails.error`), so Sentry, Honeybadger, AppSignal and friends see them
1238
+ through their Rails integration. A failed job is reported as unhandled
1239
+ with source `"robin.job"` and context `robin` (what failed), `job_class`,
1240
+ `jid`, `queue`, `retry_count` and `bid`, never its arguments. Robin's own
1241
+ errors (e.g. Redis unreachable) are reported as handled with source
1242
+ `"robin"`. The executor doesn't report jobs' exceptions itself, and
1243
+ `Rails.error` reports an exception only once, so nothing is reported
1244
+ twice. `Robin::Discard` is never reported.
1245
+ - **ActiveJob.** `config.active_job.queue_adapter = :robin` runs ActiveJob
1246
+ jobs on Robin. They can use `robin_options` (retries, tags, expiry...),
1247
+ `perform_all_later` takes one round trip, however many classes and
1248
+ queues its jobs have, and job continuations checkpoint when the server
1249
+ shuts down. Periodic jobs can be ActiveJob classes
1250
+ (`cron.register "0 3 * * *", "NightlyReportJob"`): Robin pushes them as
1251
+ `perform_later` would, on the given queue or the job's own, but without
1252
+ running enqueue callbacks. `Robin::Job` can't be included in ActiveJob
1253
+ classes: use one or the other.
1254
+ - **Transaction-aware pushes** (opt-in): jobs pushed inside Active Record
1255
+ transactions are sent once every open transaction commits, and dropped
1256
+ when it rolls back, so a job never runs before its data is visible. The
1257
+ jobs pushed in the same transaction go in one round trip.
1258
+ - **Current attributes** (opt-in): the `ActiveSupport::CurrentAttributes`
1259
+ classes you list (e.g. `Current`) are saved in each job as it's pushed
1260
+ and set while it runs, so `Current.user` or `Current.tenant` mean the
1261
+ same thing in the job as where it was pushed. They're serialized like
1262
+ ActiveJob arguments (records travel as GlobalIDs) and stored as they
1263
+ are, readable in Redis and the dashboard, so leave secrets out. Classes
1264
+ are looked up by name, so code reloading keeps working, and one renamed
1265
+ since a job was pushed is skipped.
1266
+
1267
+ ```ruby
1268
+ # config/application.rb
1269
+ config.active_job.queue_adapter = :robin
1270
+ config.robin.report_errors = true # the default; false to stop reporting to Rails.error
1271
+ config.robin.transactional_push = true # push jobs only after Active Record transactions commit
1272
+ config.robin.current_attributes = ["Current"] # carry Current.* from where a job is pushed into the job
1273
+ config.robin.notifications = false # stop publishing Robin's events as "<event>.robin" notifications
1274
+ ```
1275
+
1276
+ With `notifications` (on by default), Robin's events (see `Robin::Events`)
1277
+ are published through `ActiveSupport::Notifications` as `"<event>.robin"`:
1278
+
1279
+ ```ruby
1280
+ ActiveSupport::Notifications.subscribe("job.expired.robin") do |event|
1281
+ Rails.logger.info("expired before it ran: #{event.payload[:job]["jid"]}")
1282
+ end
1283
+ ```
1284
+
1285
+ Outside Rails' configuration, the same pieces are available directly:
1286
+ `Robin.config.push_deferrer = Robin::Rails::TransactionalPush`,
1287
+ `Robin.config.error_handlers << Robin::Rails::ErrorReporter.new(Rails.error)`,
1288
+ `Robin.config.event_handlers << Robin::Rails::Notifications` and
1289
+ `Robin::Rails::CurrentAttributes.install(Robin.config, ["Current"])`.
1290
+
1291
+ ## Testing
1292
+
1293
+ ```ruby
1294
+ require "robin/testing" # fake mode by default
1295
+
1296
+ SendReceipt.perform_async(1)
1297
+ assert_equal 1, SendReceipt.jobs.size
1298
+ SendReceipt.drain # run them
1299
+ Robin::Testing.inline! { SendReceipt.perform_async(1) } # run immediately
1300
+ Robin::Testing.disable! { ... } # talk to real Redis
1301
+ ```
1302
+
1303
+ A mode set with a block applies to the current thread only, so tests can
1304
+ run in parallel; without a block it's the default for every thread.
1305
+
1306
+ ## The data API
1307
+
1308
+ Everything in the dashboard is also available from Ruby:
1309
+
1310
+ ```ruby
1311
+ stats = Robin::Stats.new
1312
+ stats.processed; stats.failed; stats.enqueued; stats.queues; stats.history(days: 7)
1313
+
1314
+ queue = Robin::Queue.new("mailers")
1315
+ queue.size; queue.latency; queue.pause!
1316
+ queue.each { |job| job.delete if job.args.first == 42 }
1317
+ queue.delete_by_class("ReportJob") # inside Redis, a slice at a time
1318
+
1319
+ Robin::RetrySet.new.scan("NoMethodError").each(&:retry!)
1320
+ Robin::DeadSet.new.retry_all
1321
+ Robin::ScheduledSet.new.find_job(jid)&.add_to_queue
1322
+ Robin::JobLocator.new.locate(jid) # running, scheduled, retrying, dead or queued
1323
+ Robin::ProcessSet.new.each(&:quiet!)
1324
+ Robin::ProcessSet.new.first.resize!(20, capsule: "default") # workers, from its next heartbeat
1325
+ Robin::WorkSet.new.each { |identity, thread, work| ... }
1326
+ Robin::Metrics::Query.new.recent(minutes: 60)
1327
+ ```
1328
+
1329
+ `delete_by_class` matches the class the dashboard shows (an ActiveJob's
1330
+ own class, not the wrapper's) and returns how many jobs it deleted; the
1331
+ dashboard does the same when the jobs you've selected on a queue's page
1332
+ share a class. Looking a job up by jid also searches queues inside Redis,
1333
+ a slice at a time.
1334
+ With `config.jid_index = true`, Robin also remembers which queue each job
1335
+ was pushed to (for a week, or the number of seconds you give), so lookups
1336
+ search that queue first: one small key per job, in exchange for fast
1337
+ lookups in large backlogs.
1338
+
1339
+ ## Performance
1340
+
1341
+ Robin never gets slower: a change that makes it slower at any of its
1342
+ benchmarks fails CI, whatever else it improves, and is backed out or
1343
+ fixed at once. That's [WebKit's performance policy](https://webkit.org/performance/),
1344
+ enforced by [Duel](https://github.com/sferik/duel-ruby), which runs
1345
+ Robin's benchmarks ([`duel/`](duel/)) on each change and on the code it
1346
+ changes, side by side, and fails when the change is slower at any of
1347
+ them: by CPU time, instructions, Redis CPU, allocations, round trips,
1348
+ Redis commands or peak memory. Each night it also compares `main` with
1349
+ the last release, so small slowdowns can't add up unnoticed.
1350
+ [CONTRIBUTING.md](CONTRIBUTING.md#performance-never-slower) has the
1351
+ policy and how the guards tell a real regression from noise.
1352
+
1353
+ On an Apple M1 Max with local Redis, one CRuby process (YJIT) with 10
1354
+ threads runs about 120,000 no-op jobs per second, a cluster of processes
1355
+ about 280,000 against one Redis, and `perform_bulk` pushes about 360,000
1356
+ jobs per second. Workers asking for jobs at the same time share one round
1357
+ trip to Redis, so under load a job costs a fraction of one.
1358
+
1359
+ Against Sidekiq 8.1 OSS on the same machine and Redis
1360
+ (`rake bench:compare`), Robin is many times faster at many operations:
1361
+
1362
+ | | Robin vs Sidekiq |
1363
+ |---|--:|
1364
+ | Processing jobs | 5.1x faster |
1365
+ | Processing jobs with Redis 1 ms away | 8.8x faster |
1366
+ | A cluster of processes on one Redis | 4.8x faster |
1367
+ | Moving scheduled jobs onto queues when due | 61x faster |
1368
+ | Requeuing retries after an outage | 28x faster |
1369
+ | Redis CPU per job | 8x faster |
1370
+ | Jobs lost when a server is killed | 0 vs 10 |
1371
+
1372
+ [bench/RESULTS.md](bench/RESULTS.md) has the detailed results on Ruby 3.4,
1373
+ Ruby 4.0 and JRuby, what each benchmark tests and how they were run.
1374
+
1375
+ The optional C extension encodes commands, then sends them, waits for the
1376
+ replies and reads them in one release of the GVL, checking for dropped
1377
+ connections in the same call. It also parses replies 3–10x faster than pure
1378
+ Ruby, which speeds up the dashboard and the data API on large queues. The
1379
+ Java extension parses, encodes and reads on JRuby and keeps the connection
1380
+ pool's bookkeeping lock-free.
1381
+
1382
+ ## Redis layout
1383
+
1384
+ Every key starts with `config.key_prefix` (default `robin:`), so Robin can
1385
+ share a database with other data. Robin uses Lua scripts for its atomic
1386
+ operations and doesn't support Redis Cluster; use a single primary, with a
1387
+ replica for failover if you need one.
1388
+
1389
+ The keys a job's life runs through (`queue:<name>`, `queues`, `schedule`,
1390
+ `retry`, `dead`, `paused`, `processes`, `stat:*`, and each running
1391
+ process's record at its identity) keep the formats common Ruby job tools
1392
+ share: queues are lists pushed at the head and taken from the tail; the
1393
+ schedule, retry and dead sets are scored in epoch seconds; jobs are JSON
1394
+ with `created_at` and `enqueued_at` in epoch milliseconds, a 0-based
1395
+ `retry_count` and a compressed `error_backtrace`. Robin's own keys
1396
+ (batches, limiters, metrics, in-flight jobs...) can get a prefix of
1397
+ their own with `config.internal_key_prefix`, so with `key_prefix = ""` the
1398
+ job keys sit at the top level, where other tools look for them.
1399
+
1400
+ A process keeps each job it has fetched in a list of its own,
1401
+ `inflight:<identity>:<n>`, registered in the set `inflight:<identity>`
1402
+ (listed in `inflights`) before a job is first moved into it; a key is
1403
+ used again once its job is acknowledged. Recovery still reads the single
1404
+ `inflight:<identity>` list that processes of earlier development versions
1405
+ kept their jobs in.
1406
+
1407
+ ## Development
1408
+
1409
+ ```sh
1410
+ bundle install
1411
+ rake compile # build the C extension (optional); on JRuby, the Java one
1412
+ rake test # needs a local Redis; uses db 15 (ROBIN_TEST_REDIS_URL)
1413
+ # one process per test file, TEST_JOBS at a time
1414
+ # (default: one per CPU); rake test:serial for one process
1415
+ rake test:rails # just the Rails integration's tests (test/rails/)
1416
+ rake web:build # rebuild the dashboard (needs bun)
1417
+ cd web && bun run dev # dashboard watch mode
1418
+ rake bench
1419
+ rake bench:compare # Robin vs Sidekiq (installs bench/compare/Gemfile)
1420
+ rake perf # fails if this tree is slower than origin/main (Duel; needs redis-server)
1421
+ ```
1422
+
1423
+ To try things out by hand, [`demo/`](demo/README.md) is a small app with
1424
+ sample jobs and seed data: `cd demo && bin/seed && bin/dev`.
1425
+
1426
+ Robin holds itself to 100% test coverage and 100% mutation coverage, on
1427
+ the Ruby, C, Java and TypeScript sides. CI enforces it:
1428
+
1429
+ | Check | Command |
1430
+ |---|---|
1431
+ | Ruby lint | `bundle exec rubocop` and `bundle exec standardrb` |
1432
+ | Ruby tests (MRI and JRuby) | `rake test` |
1433
+ | Ruby coverage: 100% lines, branches and methods | `COVERAGE_STRICT=1 rake test` |
1434
+ | Ruby mutation coverage: 100% | `bundle exec mutant run` |
1435
+ | Ruby types: RBS signatures (`sig/`) and Steep, strict (CRuby) | `rake steep` |
1436
+ | Performance: never slower than the baseline at any benchmark ([Duel](https://github.com/sferik/duel-ruby); [CONTRIBUTING.md](CONTRIBUTING.md#performance-never-slower)) | `rake perf` |
1437
+ | C lint: clang-format, clang-tidy, `-Werror` | `rake c:lint` |
1438
+ | C coverage: 100% lines, branches and functions | `rake c:coverage` (needs gcovr) |
1439
+ | Java lint: google-java-format, `javac -Xlint:all -Werror` | `rake java:lint` |
1440
+ | Java coverage: 100% lines, branches and methods (JaCoCo) | `rake java:coverage` |
1441
+ | Java mutation coverage: 100% (PIT) | `rake java:mutation` |
1442
+ | Frontend lint and types | `cd web && bun run lint && bun run typecheck` |
1443
+ | Frontend coverage: 100% | `cd web && bun run test:coverage` |
1444
+ | Frontend mutation coverage: 100% | `cd web && bun run test:mutation` |
1445
+
1446
+ The Java tasks run under JRuby with a JDK 21 (or set `JRUBY_JAR` to JRuby's
1447
+ `lib/jruby.jar`); they download their pinned, checksummed tools into
1448
+ `tmp/java-tools`. The java-platform gem is built under JRuby with
1449
+ `rake java:compile` and then `ROBIN_PLATFORM=java gem build robin.gemspec`.
1450
+
1451
+ **Translating the dashboard.** Messages are written in English in the
1452
+ code, wrapped in `t()` (or `tn()` for plurals), and each language's
1453
+ `web/src/locales/<code>.ts` maps them to translations. `bun run
1454
+ scripts/messages.ts de` (in `web/`) lists what German is missing, and the
1455
+ tests check that every catalogue has every message, keeps its
1456
+ placeholders and gives each plural form its language uses.
1457
+
1458
+ Tests pin how many Redis round trips Robin makes with `assert_roundtrips`
1459
+ (from `test/support/roundtrips.rb`), so a change that adds one fails:
1460
+
1461
+ ```ruby
1462
+ assert_roundtrips(1) { Robin::Client.push("class" => "Thing", "args" => [1]) }
1463
+ ```
1464
+
1465
+ It counts the calls, pipelines and transactions the block's thread sends
1466
+ (`all_threads: true` for every thread's), not connecting or reloading a
1467
+ Lua script Redis lost, takes an Integer or a Range, returns the block's
1468
+ value, and lists the commands of each round trip when the count is wrong.
1469
+
1470
+ Assertions are strict (minitest-strict): `assert`, `refute` and the
1471
+ predicate and operator assertions take only `true` and `false`, so a
1472
+ method that returns `nil` where it means `false` fails its test. On CRuby,
1473
+ `test/allocations_test.rb` holds the paths every job takes (pushing,
1474
+ fetching, running, the middleware chain, metrics, logging) to allocation
1475
+ budgets with minitest-memory's `assert_allocations`; re-measure them with
1476
+ `ROBIN_MEASURE_ALLOCATIONS=1`.
1477
+
1478
+ Mutation testing starts many test processes at once. Point
1479
+ `ROBIN_TEST_REDIS_URL` at a Redis listening on a Unix socket (e.g.
1480
+ `unix:///tmp/redis.sock?db=15`) so they don't exhaust local TCP ports.
1481
+
1482
+ ## License
1483
+
1484
+ Robin is released under the [GNU Lesser General Public License v3.0](LICENSE)
1485
+ (see also [COPYING](COPYING) for the GPL v3 it builds on). You can use Robin
1486
+ in proprietary applications. Changes to Robin itself must be shared under
1487
+ the same license.
1488
+
1489
+ The Robin mascot and logos in `assets/` are part of the project and
1490
+ licensed under the same terms.