robin 0.0.4 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/robin_resp/extconf.rb +29 -0
  13. data/ext/robin_resp/robin_resp.c +645 -0
  14. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  15. data/lib/generators/robin/install/install_generator.rb +39 -0
  16. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  17. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  18. data/lib/generators/robin/job/job_generator.rb +78 -0
  19. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  20. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  21. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  22. data/lib/puma/plugin/robin.rb +131 -0
  23. data/lib/robin/active_job.rb +144 -0
  24. data/lib/robin/api.rb +777 -0
  25. data/lib/robin/batch.rb +700 -0
  26. data/lib/robin/cancellation.rb +62 -0
  27. data/lib/robin/cli.rb +325 -0
  28. data/lib/robin/client.rb +355 -21
  29. data/lib/robin/config.rb +454 -4
  30. data/lib/robin/cron.rb +325 -0
  31. data/lib/robin/debounce.rb +108 -0
  32. data/lib/robin/deploy.rb +65 -0
  33. data/lib/robin/embedded.rb +53 -0
  34. data/lib/robin/encryption.rb +147 -0
  35. data/lib/robin/events.rb +60 -0
  36. data/lib/robin/expiry.rb +41 -0
  37. data/lib/robin/fetcher.rb +441 -0
  38. data/lib/robin/gvl.rb +62 -0
  39. data/lib/robin/health.rb +76 -0
  40. data/lib/robin/iterable_job.rb +294 -0
  41. data/lib/robin/job.rb +258 -0
  42. data/lib/robin/job_class.rb +157 -0
  43. data/lib/robin/job_logger.rb +121 -0
  44. data/lib/robin/launcher.rb +658 -0
  45. data/lib/robin/leader.rb +86 -0
  46. data/lib/robin/limiter.rb +639 -0
  47. data/lib/robin/manager.rb +125 -0
  48. data/lib/robin/metrics.rb +303 -0
  49. data/lib/robin/middleware.rb +165 -0
  50. data/lib/robin/opentelemetry.rb +88 -0
  51. data/lib/robin/periodic.rb +327 -0
  52. data/lib/robin/processor.rb +317 -0
  53. data/lib/robin/profiler.rb +140 -0
  54. data/lib/robin/progress.rb +46 -0
  55. data/lib/robin/rails/app_loader.rb +23 -0
  56. data/lib/robin/rails/current_attributes.rb +88 -0
  57. data/lib/robin/rails/error_reporter.rb +31 -0
  58. data/lib/robin/rails/notifications.rb +22 -0
  59. data/lib/robin/rails/railtie.rb +87 -0
  60. data/lib/robin/rails/transactional_push.rb +49 -0
  61. data/lib/robin/rails.rb +23 -0
  62. data/lib/robin/recovery.rb +135 -0
  63. data/lib/robin/redis/connection.rb +639 -0
  64. data/lib/robin/redis/pool.rb +190 -0
  65. data/lib/robin/redis/resp.rb +77 -0
  66. data/lib/robin/reliable_push.rb +153 -0
  67. data/lib/robin/retrier.rb +284 -0
  68. data/lib/robin/scheduler.rb +102 -0
  69. data/lib/robin/scripts.rb +331 -0
  70. data/lib/robin/statsd.rb +135 -0
  71. data/lib/robin/swarm.rb +319 -0
  72. data/lib/robin/systemd.rb +47 -0
  73. data/lib/robin/testing.rb +158 -0
  74. data/lib/robin/top.rb +178 -0
  75. data/lib/robin/unique.rb +198 -0
  76. data/lib/robin/version.rb +5 -0
  77. data/lib/robin/warmup.rb +94 -0
  78. data/lib/robin/web/api.rb +580 -0
  79. data/lib/robin/web/batch_status.rb +44 -0
  80. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  81. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  82. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  83. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  84. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  86. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  87. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  88. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  89. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  90. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  91. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  92. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  93. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  94. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  95. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  96. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  97. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  98. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  99. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  100. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  101. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  102. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  103. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  104. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  105. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  106. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  107. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  108. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  109. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  110. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  111. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  112. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  113. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  114. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  115. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  116. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  117. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  118. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  119. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  120. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  121. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  122. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  123. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  124. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  125. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  126. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  127. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  128. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  129. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  130. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  131. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  132. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  133. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  134. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  135. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  136. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  137. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  138. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  139. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  140. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  141. data/lib/robin/web/dist/index.html +19 -0
  142. data/lib/robin/web/extension.rb +193 -0
  143. data/lib/robin/web/permissions.rb +40 -0
  144. data/lib/robin/web/prometheus.rb +79 -0
  145. data/lib/robin/web/router.rb +150 -0
  146. data/lib/robin/web/static.rb +110 -0
  147. data/lib/robin/web.rb +196 -0
  148. data/lib/robin.rb +258 -8
  149. data/sig/manifest.yaml +9 -0
  150. data/sig/robin/api.rbs +442 -0
  151. data/sig/robin/batch.rbs +292 -0
  152. data/sig/robin/cancellation.rbs +25 -0
  153. data/sig/robin/cli.rbs +102 -0
  154. data/sig/robin/client.rbs +118 -0
  155. data/sig/robin/config.rbs +341 -0
  156. data/sig/robin/cron.rbs +149 -0
  157. data/sig/robin/debounce.rbs +36 -0
  158. data/sig/robin/deploy.rbs +24 -0
  159. data/sig/robin/embedded.rbs +25 -0
  160. data/sig/robin/encryption.rbs +49 -0
  161. data/sig/robin/events.rbs +10 -0
  162. data/sig/robin/expiry.rbs +17 -0
  163. data/sig/robin/fetcher.rbs +134 -0
  164. data/sig/robin/gvl.rbs +39 -0
  165. data/sig/robin/health.rbs +30 -0
  166. data/sig/robin/iterable_job.rbs +99 -0
  167. data/sig/robin/job.rbs +173 -0
  168. data/sig/robin/job_class.rbs +55 -0
  169. data/sig/robin/job_logger.rbs +56 -0
  170. data/sig/robin/launcher.rbs +191 -0
  171. data/sig/robin/leader.rbs +43 -0
  172. data/sig/robin/limiter.rbs +317 -0
  173. data/sig/robin/manager.rbs +54 -0
  174. data/sig/robin/metrics.rbs +122 -0
  175. data/sig/robin/middleware.rbs +106 -0
  176. data/sig/robin/opentelemetry.rbs +51 -0
  177. data/sig/robin/periodic.rbs +81 -0
  178. data/sig/robin/processor.rbs +107 -0
  179. data/sig/robin/profiler.rbs +60 -0
  180. data/sig/robin/progress.rbs +22 -0
  181. data/sig/robin/rails/app_loader.rbs +10 -0
  182. data/sig/robin/rails/current_attributes.rbs +44 -0
  183. data/sig/robin/rails/error_reporter.rbs +21 -0
  184. data/sig/robin/rails/notifications.rbs +9 -0
  185. data/sig/robin/rails/transactional_push.rbs +16 -0
  186. data/sig/robin/rails.rbs +8 -0
  187. data/sig/robin/recovery.rbs +29 -0
  188. data/sig/robin/redis/connection.rbs +224 -0
  189. data/sig/robin/redis/pool.rbs +89 -0
  190. data/sig/robin/redis/resp.rbs +29 -0
  191. data/sig/robin/reliable_push.rbs +77 -0
  192. data/sig/robin/retrier.rbs +72 -0
  193. data/sig/robin/scheduler.rbs +40 -0
  194. data/sig/robin/scripts.rbs +36 -0
  195. data/sig/robin/statsd.rbs +55 -0
  196. data/sig/robin/swarm.rbs +133 -0
  197. data/sig/robin/systemd.rbs +13 -0
  198. data/sig/robin/testing.rbs +80 -0
  199. data/sig/robin/top.rbs +79 -0
  200. data/sig/robin/unique.rbs +72 -0
  201. data/sig/robin/version.rbs +3 -0
  202. data/sig/robin/warmup.rbs +44 -0
  203. data/sig/robin/web/api.rbs +196 -0
  204. data/sig/robin/web/batch_status.rbs +25 -0
  205. data/sig/robin/web/extension.rbs +106 -0
  206. data/sig/robin/web/permissions.rbs +14 -0
  207. data/sig/robin/web/prometheus.rbs +33 -0
  208. data/sig/robin/web/router.rbs +53 -0
  209. data/sig/robin/web/static.rbs +42 -0
  210. data/sig/robin/web.rbs +140 -0
  211. data/sig/robin.rbs +116 -0
  212. data/sig/robin_resp.rbs +55 -0
  213. metadata +274 -44
  214. data/bin/robin +0 -5
  215. data/lib/robin/actions/authenticate.rb +0 -38
  216. data/lib/robin/actions/empty_tweet.rb +0 -7
  217. data/lib/robin/actions/followers.rb +0 -22
  218. data/lib/robin/actions/help.rb +0 -16
  219. data/lib/robin/actions/home_timeline.rb +0 -9
  220. data/lib/robin/actions/new_tweet.rb +0 -11
  221. data/lib/robin/actions/unknown_flag.rb +0 -11
  222. data/lib/robin/actions/user_timeline.rb +0 -19
  223. data/lib/robin/args_parser.rb +0 -70
  224. data/lib/robin/pin_authenticator.rb +0 -31
  225. data/lib/robin/user.rb +0 -36
data/lib/robin/config.rb CHANGED
@@ -1,11 +1,461 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Robin
4
+ # A capsule is a named group of worker threads with its own queues and
5
+ # concurrency. Every config has a "default" capsule; add more to isolate
6
+ # work, e.g. a single-threaded capsule for jobs that must run serially.
7
+ class Capsule
8
+ attr_reader :name, :config, :weights
9
+
10
+ # The capsule whose worker is running this thread's code (a job, say),
11
+ # or nil outside a server's workers (perform_inline, a console).
12
+ def self.current = Thread.current[:robin_capsule]
13
+
14
+ def initialize(name, config)
15
+ @name = name
16
+ @config = config
17
+ # A box, so a running capsule (frozen; see Config#freeze_capsules!)
18
+ # can still be resized.
19
+ @workers = {count: 5}
20
+ self.queues = "default"
21
+ end
22
+
23
+ # How many workers the capsule runs: as configured, and once it runs,
24
+ # as Launcher#resize last set it.
25
+ def concurrency = @workers.fetch(:count)
26
+
27
+ def concurrency=(count)
28
+ raise FrozenError.new("can't modify frozen #{self.class}: #{inspect}", receiver: self) if frozen?
29
+
30
+ @workers[:count] = count
31
+ end
32
+
33
+ # A running capsule's new worker count (see Manager#resize).
34
+ def resized(count)
35
+ @workers[:count] = count
36
+ end
37
+
38
+ # Accepts plain names ("critical"), weighted names ("critical,3" or
39
+ # ["critical", 3]). With no weights the order is strict priority; with
40
+ # weights a queue is checked first proportionally to its weight.
41
+ def queues=(list)
42
+ @weights = {}
43
+ Array(list).each do |entry|
44
+ name, weight = entry.instance_of?(Array) ? entry : entry.to_s.split(",", 2)
45
+ name = name.to_s.strip
46
+ raise ArgumentError, "queue #{name.inspect} is listed twice; give it one weight instead" if @weights.key?(name)
47
+
48
+ # (Steep loses track of weight's type through Array(list).)
49
+ @weights[name] = Integer(weight || 0) # steep:ignore ArgumentTypeMismatch
50
+ end
51
+ @strict = @weights.values.all?(&:zero?)
52
+ @expanded = @weights.flat_map { |q, w| [q] * [w, 1].max }
53
+ end
54
+
55
+ def queues
56
+ @weights.keys
57
+ end
58
+
59
+ def strict?
60
+ @strict
61
+ end
62
+
63
+ # Queue names in the order they should be checked for the next fetch.
64
+ def queue_order
65
+ return queues if @strict
66
+
67
+ @expanded.shuffle.uniq
68
+ end
69
+
70
+ def freeze
71
+ @weights.freeze
72
+ super
73
+ end
74
+ end
75
+
2
76
  class Config
3
- def self.key
4
- 'WgyUHBHODrfuzwGYdfukw'
77
+ # Logs the error, its context and the top of its backtrace as one
78
+ # entry, so they stay together in a combined log stream.
79
+ DEFAULT_ERROR_HANDLER = lambda do |ex, context, config|
80
+ config.logger.warn do
81
+ ["#{ex.class}: #{ex.message}", (Robin.dump_json(context) unless context.empty?), *ex.backtrace&.first(20)]
82
+ .compact.join("\n")
83
+ end
5
84
  end
6
85
 
7
- def self.secret
8
- '7MdWF7SXLTICfToz118RhHV07eMXKakbWmyIhlKE'
86
+ # Robin's connections name themselves (CLIENT SETNAME), so they're easy
87
+ # to tell apart in CLIENT LIST. A +name:+ in the Redis options wins.
88
+ CONNECTION_DEFAULTS = {name: "robin"}.freeze
89
+
90
+ # Whether servers warm up the JIT by default (see #jit_warmup): off
91
+ # CRuby.
92
+ JIT_WARMUP = !RUBY_ENGINE.eql?("ruby")
93
+
94
+ LIFECYCLE_EVENTS = %i[startup quiet shutdown leader follower heartbeat exit fork].freeze
95
+
96
+ attr_accessor :timeout, :poll_interval, :max_retries, :dead_max_jobs, :dead_timeout,
97
+ :strict_args, :key_prefix, :tag, :labels, :environment,
98
+ :heartbeat_interval, :process_ttl, :max_recoveries, :backtrace_lines,
99
+ :encryption_keys, :max_rate_limit_retries, :statsd, :memory_limit_mb,
100
+ :metrics_retention_days, :reliable_push, :shutdown_requeue, :job_wrapper,
101
+ :health_bind
102
+
103
+ # How long to wait before running a job again that hit a rate limit:
104
+ # a callable taking (count, error, payload), where count is how many
105
+ # times the job has hit a limit and error the Robin::Limiter::OverLimit,
106
+ # returning seconds (or nil for the default backoff). A limiter's own
107
+ # +backoff:+ comes first.
108
+ attr_accessor :rate_limit_backoff
109
+
110
+ # The class of the batch status batch callbacks get (default
111
+ # Robin::Batch::Status): a subclass, taking (bid, config:) and
112
+ # answering exists?, may add methods of its own.
113
+ attr_writer :batch_status_class
114
+
115
+ def batch_status_class = @batch_status_class || Batch::Status
116
+
117
+ # Whether this process may become the cluster's leader (default true).
118
+ # Servers that shouldn't run the leader's duties (periodic jobs, crash
119
+ # recovery) set it false.
120
+ attr_accessor :leader_candidate
121
+
122
+ # Callables run with each job recovered from a crashed process: its
123
+ # payload, and whether it was a poison pill moved to the dead set
124
+ # (see max_recoveries) rather than put back on its queue.
125
+ attr_reader :recovery_handlers
126
+
127
+ # Callables run with each of Robin's events, such as "job.expired" or
128
+ # "redis.slow": the event's name and a Hash about it. Robin::Events
129
+ # lists them.
130
+ attr_reader :event_handlers
131
+
132
+ # Whether a server measures GVL contention (see Robin::GVL; default
133
+ # true). Where it can (with the C extension), each heartbeat
134
+ # records it in the process's record, for the dashboard.
135
+ attr_accessor :gvl_monitoring
136
+
137
+ # Replaces the retry backoff: a callable (failures, exception, payload,
138
+ # custom) returning the seconds before a failed job's retry, where
139
+ # failures counts from 1 and custom is the job class's robin_retry_in
140
+ # answer in seconds (or nil). Robin adds no jitter to what it returns.
141
+ # nil (the default) is Robin's own backoff.
142
+ attr_accessor :retry_backoff
143
+
144
+ # Whether a standalone server collects its garbage once it has gone
145
+ # idle: false (the default), or the fewest jobs that must have
146
+ # finished since the last collection (true is 1). Once no job is
147
+ # running and none has finished for a second, the next heartbeat runs
148
+ # a full GC, so the next burst of jobs doesn't pay for the last one's
149
+ # garbage. A full GC stops every thread of the process while it runs
150
+ # (tens of milliseconds for a large heap; on JRuby, the whole JVM), a
151
+ # job that comes meanwhile waits for it, and it costs CPU every time
152
+ # the process goes quiet, which for bursty work may be every few
153
+ # seconds. Embedded servers never do, as they share their process
154
+ # with the app.
155
+ attr_reader :idle_gc
156
+
157
+ def idle_gc=(least)
158
+ least = 1 if least == true
159
+ if least && !(least.instance_of?(Integer) && least >= 1)
160
+ raise ArgumentError, "idle_gc must be true, false or a number of jobs (at least 1), not #{least.inspect}"
161
+ end
162
+
163
+ @idle_gc = least || false
9
164
  end
165
+
166
+ # Whether a server warms up the JIT before it starts its workers (see
167
+ # Robin::Warmup; default false on CRuby, where YJIT warms up quickly
168
+ # anyway, and true elsewhere: on JRuby it takes about a second and a
169
+ # half).
170
+ attr_accessor :jit_warmup
171
+
172
+ # Seconds an idle worker blocks waiting for a job (default 2). Longer
173
+ # means fewer idle round trips; shorter means idle workers notice a
174
+ # shutdown sooner, and (with more queues than workers) a job waits less
175
+ # in a queue no worker is blocked on.
176
+ attr_accessor :fetch_timeout
177
+
178
+ # The most jobs one fetch takes from Redis (default 10): a worker
179
+ # fetching takes one for every worker of its capsule waiting for one,
180
+ # up to this, in a single round trip. 1 fetches one job at a time.
181
+ attr_reader :fetch_batch
182
+
183
+ def fetch_batch=(count)
184
+ count = Integer(count)
185
+ raise ArgumentError, "fetch_batch must be at least 1, not #{count}" if count < 1
186
+
187
+ @fetch_batch = count
188
+ end
189
+
190
+ # Seconds an iterable job runs before it checkpoints and requeues
191
+ # itself so other jobs get a turn (nil, the default: no limit). A job's
192
+ # max_iteration_runtime robin_option overrides it.
193
+ attr_accessor :max_iteration_runtime
194
+
195
+ # Trims the backtraces stored with failed jobs, e.g. dropping gem
196
+ # frames: called with the backtrace (an array of lines), returns the
197
+ # lines to keep. The Rails integration uses Rails.backtrace_cleaner.
198
+ #
199
+ # config.backtrace_cleaner = ->(lines) { lines.reject { |l| l.include?("/gems/") } }
200
+ attr_accessor :backtrace_cleaner
201
+
202
+ # How long the jid index remembers a job's queue when it's just true.
203
+ JID_INDEX_TTL = 7 * 86_400
204
+
205
+ # Seconds the queue each pushed job went to is remembered (true means
206
+ # a week; nil, the default, turns it off), so finding a job by jid
207
+ # (Robin::JobLocator, the web UI) searches that queue first instead of
208
+ # every queue. Costs a small key per job.
209
+ attr_reader :jid_index
210
+
211
+ def jid_index=(value)
212
+ @jid_index = value.equal?(true) ? JID_INDEX_TTL : value && Integer(value)
213
+ end
214
+
215
+ # The port of a server's health check endpoint (see Robin::HealthServer);
216
+ # nil (the default, unless ROBIN_HEALTH_PORT is set) means none.
217
+ attr_reader :health_port
218
+
219
+ def health_port=(port)
220
+ @health_port = port && Integer(port)
221
+ end
222
+
223
+ # A callable that may defer delivering pushed jobs, e.g. until a
224
+ # database transaction commits (the Rails integration uses it for
225
+ # Active Record). It receives the normalized payloads and a block that
226
+ # delivers them, and returns truthy when it took over delivery
227
+ # (calling the block later, or right away); falsy means "deliver now".
228
+ # Jobs pushed inside a batch are never deferred.
229
+ #
230
+ # config.push_deferrer = lambda do |_payloads, &deliver|
231
+ # next false unless MyDB.in_transaction?
232
+ # MyDB.after_commit(&deliver)
233
+ # true
234
+ # end
235
+ attr_accessor :push_deferrer
236
+
237
+ attr_writer :logger, :redis_pool
238
+ attr_reader :capsules, :error_handlers, :death_handlers, :lifecycle, :periodic_blocks
239
+
240
+ def initialize
241
+ @redis_options = {}
242
+ @capsules = {}
243
+ @timeout = 25
244
+ @poll_interval = 5
245
+ @max_retries = 20
246
+ @dead_max_jobs = 10_000
247
+ @dead_timeout = 180 * 86_400
248
+ @strict_args = :raise
249
+ @key_prefix = "robin:"
250
+ @labels = []
251
+ @environment = ENV["APP_ENV"] || ENV["RACK_ENV"] || "development"
252
+ @heartbeat_interval = 5
253
+ @fetch_timeout = 2
254
+ @fetch_batch = 10
255
+ @process_ttl = 60
256
+ @max_recoveries = 3
257
+ @backtrace_lines = 30
258
+ @max_rate_limit_retries = 20
259
+ @metrics_retention_days = 90
260
+ @reliable_push = false
261
+ @shutdown_requeue = true
262
+ @gvl_monitoring = true
263
+ @idle_gc = false
264
+ @jit_warmup = JIT_WARMUP
265
+ self.health_port = ENV["ROBIN_HEALTH_PORT"]
266
+ @health_bind = "0.0.0.0"
267
+ @error_handlers = [DEFAULT_ERROR_HANDLER]
268
+ @death_handlers = []
269
+ @recovery_handlers = []
270
+ @event_handlers = []
271
+ @leader_candidate = true
272
+ @lifecycle = LIFECYCLE_EVENTS.to_h { |e| [e, []] }
273
+ @periodic_blocks = []
274
+ @client_chain = Middleware::Chain.new(self)
275
+ @server_chain = Middleware::Chain.new(self)
276
+ @capsules["default"] = Capsule.new("default", self)
277
+ install_default_middleware
278
+ end
279
+
280
+ # Built-in middleware powering per-job options (unique_for, debounce,
281
+ # encrypt, expires_in, batches, rate limits, profile). Each is a no-op for jobs
282
+ # that don't use its option.
283
+ def install_default_middleware
284
+ @client_chain.add(Unique::ClientMiddleware)
285
+ @client_chain.add(Debounce::ClientMiddleware)
286
+ @client_chain.add(Batch::ClientMiddleware)
287
+ @client_chain.add(Encryption::ClientMiddleware) # last: sees plaintext args
288
+ @server_chain.add(Encryption::ServerMiddleware) # first: decrypts for everyone
289
+ @server_chain.add(Expiry::ServerMiddleware)
290
+ @server_chain.add(Limiter::ServerMiddleware)
291
+ @server_chain.add(Unique::ServerMiddleware)
292
+ @server_chain.add(Debounce::ServerMiddleware)
293
+ @server_chain.add(Batch::ServerMiddleware)
294
+ @server_chain.add(Profiler::ServerMiddleware) # last: profiles just the job
295
+ end
296
+
297
+ def default_capsule
298
+ @capsules.fetch("default")
299
+ end
300
+
301
+ def concurrency = default_capsule.concurrency
302
+
303
+ def concurrency=(n)
304
+ default_capsule.concurrency = n
305
+ end
306
+
307
+ def queues = default_capsule.queues
308
+
309
+ def queues=(list)
310
+ default_capsule.queues = list
311
+ end
312
+
313
+ def capsule(name)
314
+ cap = (@capsules[name.to_s] ||= Capsule.new(name.to_s, self))
315
+ yield cap if block_given?
316
+ cap
317
+ end
318
+
319
+ def total_concurrency
320
+ @capsules.values.sum(&:concurrency)
321
+ end
322
+
323
+ # Called when a server boots. Capsules (their queues and concurrency)
324
+ # only take effect then, so changing them afterwards raises FrozenError
325
+ # instead of silently doing nothing.
326
+ def freeze_capsules!
327
+ @capsules.each_value(&:freeze)
328
+ @capsules.freeze
329
+ end
330
+
331
+ def redis=(options)
332
+ @redis_options = options.transform_keys(&:to_sym)
333
+ @redis_pool&.shutdown
334
+ @redis_pool = nil
335
+ @housekeeping_pool&.shutdown
336
+ @housekeeping_pool = nil
337
+ end
338
+
339
+ attr_reader :redis_options
340
+
341
+ # The shared connection pool. Servers size it to fit every worker
342
+ # thread plus the housekeeping threads.
343
+ def redis_pool
344
+ @redis_pool ||= begin
345
+ opts = @redis_options.dup
346
+ size = opts.delete(:pool_size) || default_pool_size(total_concurrency)
347
+ pool_timeout = opts.delete(:pool_timeout) || 3.0
348
+ idle_timeout = opts.delete(:idle_timeout)
349
+ Redis::Pool.new(size: size, timeout: pool_timeout, idle_timeout: idle_timeout) { Redis::Connection.new(**CONNECTION_DEFAULTS, **opts) }
350
+ end
351
+ end
352
+
353
+ # Makes the shared pool big enough for +workers+ worker threads: one
354
+ # Robin sized grows; one whose size was set (:pool_size) doesn't.
355
+ # Returns whether it's big enough.
356
+ def fit_redis_pool(workers)
357
+ redis_pool.grow(default_pool_size(workers)) unless @redis_options.key?(:pool_size)
358
+ redis_pool.size >= workers + 2
359
+ end
360
+
361
+ # A small pool reserved for a server's own housekeeping (heartbeat,
362
+ # leader chores), so busy workers can never starve the heartbeat of a
363
+ # connection and get the process mistaken for dead.
364
+ def housekeeping_pool
365
+ @housekeeping_pool ||= begin
366
+ opts = @redis_options.except(:pool_size, :pool_timeout, :idle_timeout)
367
+ Redis::Pool.new(size: 2) { Redis::Connection.new(**CONNECTION_DEFAULTS, **opts) }
368
+ end
369
+ end
370
+
371
+ def redis(&)
372
+ redis_pool.with(&)
373
+ end
374
+
375
+ # The keys a job's life runs through, which other tools may read (or
376
+ # share): queues, the schedule, retry and dead sets, the paused set,
377
+ # the process list and the job counters. Their names start with
378
+ # key_prefix; every other key's, with internal_key_prefix.
379
+ SHARED_KEYS = %w[queue queues schedule retry dead paused processes stat].freeze
380
+
381
+ # The prefix of Robin's own keys (batches, limiters, metrics, in-flight
382
+ # lists...): key_prefix unless set. With key_prefix "" (keys named
383
+ # like other tools' with no prefix) this keeps Robin's own apart.
384
+ attr_writer :internal_key_prefix
385
+
386
+ def internal_key_prefix = @internal_key_prefix || @key_prefix
387
+
388
+ def key(*parts)
389
+ "#{SHARED_KEYS.include?(parts.first) ? @key_prefix : internal_key_prefix}#{parts.join(":")}"
390
+ end
391
+
392
+ # A running process's record (a hash), named by its identity alone;
393
+ # its running jobs are at "<key>:work" and signals for it at
394
+ # "<key>-signals".
395
+ def process_key(identity) = "#{@key_prefix}#{identity}"
396
+
397
+ def logger
398
+ @logger ||= Logger.new($stdout, level: Logger::INFO).tap do |l|
399
+ l.formatter = JobLogger::Formatter.new
400
+ end
401
+ end
402
+
403
+ def client_middleware
404
+ yield @client_chain if block_given?
405
+ @client_chain
406
+ end
407
+
408
+ def server_middleware
409
+ yield @server_chain if block_given?
410
+ @server_chain
411
+ end
412
+
413
+ # Register a lifecycle callback: :startup, :quiet, :shutdown, :leader,
414
+ # :follower, :heartbeat or :exit.
415
+ def on(event, &block)
416
+ raise ArgumentError, "unknown event #{event.inspect}" unless @lifecycle.key?(event)
417
+
418
+ @lifecycle.fetch(event) << block
419
+ end
420
+
421
+ # Runs the callbacks for +event+. Errors are reported; with +reraise+
422
+ # (used for :startup, so a server never runs half set up) they're also
423
+ # raised.
424
+ def fire_event(event, reverse: false, reraise: false)
425
+ blocks = @lifecycle.fetch(event)
426
+ blocks = blocks.reverse if reverse
427
+ blocks.each do |block|
428
+ block.call
429
+ rescue => e
430
+ handle_exception(e, {context: "lifecycle event #{event}"})
431
+ raise if reraise
432
+ end
433
+ end
434
+
435
+ # Register periodic (cron) jobs. The block receives a Robin::Periodic
436
+ # registry; only the elected leader enqueues them.
437
+ #
438
+ # config.periodic do |cron|
439
+ # cron.register "*/5 * * * *", "CleanupJob"
440
+ # end
441
+ def periodic(&block)
442
+ @periodic_blocks << block
443
+ end
444
+
445
+ def handle_exception(ex, context = {})
446
+ @error_handlers.each do |handler|
447
+ handler.call(ex, context, self)
448
+ rescue => e
449
+ logger.error "!!! error handler raised: #{e.class}: #{e}"
450
+ end
451
+ end
452
+
453
+ def inspect
454
+ "#<Robin::Config concurrency=#{total_concurrency} capsules=#{@capsules.keys} prefix=#{@key_prefix.inspect}>"
455
+ end
456
+
457
+ private
458
+
459
+ def default_pool_size(workers) = Robin.server? ? workers + 5 : 10
10
460
  end
11
461
  end