robin 0.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/java/robin_resp/Blocking.java +27 -0
  13. data/ext/java/robin_resp/NativeIO.java +229 -0
  14. data/ext/java/robin_resp/NativeRESP.java +279 -0
  15. data/ext/java/robin_resp/NativeSlots.java +133 -0
  16. data/ext/java/robin_resp/RobinRespService.java +32 -0
  17. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  18. data/lib/generators/robin/install/install_generator.rb +39 -0
  19. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  20. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  21. data/lib/generators/robin/job/job_generator.rb +78 -0
  22. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  23. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  24. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  25. data/lib/puma/plugin/robin.rb +131 -0
  26. data/lib/robin/active_job.rb +144 -0
  27. data/lib/robin/api.rb +777 -0
  28. data/lib/robin/batch.rb +700 -0
  29. data/lib/robin/cancellation.rb +62 -0
  30. data/lib/robin/cli.rb +325 -0
  31. data/lib/robin/client.rb +372 -0
  32. data/lib/robin/config.rb +461 -0
  33. data/lib/robin/cron.rb +325 -0
  34. data/lib/robin/debounce.rb +108 -0
  35. data/lib/robin/deploy.rb +65 -0
  36. data/lib/robin/embedded.rb +53 -0
  37. data/lib/robin/encryption.rb +147 -0
  38. data/lib/robin/events.rb +60 -0
  39. data/lib/robin/expiry.rb +41 -0
  40. data/lib/robin/fetcher.rb +441 -0
  41. data/lib/robin/gvl.rb +62 -0
  42. data/lib/robin/health.rb +76 -0
  43. data/lib/robin/iterable_job.rb +294 -0
  44. data/lib/robin/job.rb +258 -0
  45. data/lib/robin/job_class.rb +157 -0
  46. data/lib/robin/job_logger.rb +121 -0
  47. data/lib/robin/launcher.rb +658 -0
  48. data/lib/robin/leader.rb +86 -0
  49. data/lib/robin/limiter.rb +639 -0
  50. data/lib/robin/manager.rb +125 -0
  51. data/lib/robin/metrics.rb +303 -0
  52. data/lib/robin/middleware.rb +165 -0
  53. data/lib/robin/opentelemetry.rb +88 -0
  54. data/lib/robin/periodic.rb +327 -0
  55. data/lib/robin/processor.rb +317 -0
  56. data/lib/robin/profiler.rb +140 -0
  57. data/lib/robin/progress.rb +46 -0
  58. data/lib/robin/rails/app_loader.rb +23 -0
  59. data/lib/robin/rails/current_attributes.rb +88 -0
  60. data/lib/robin/rails/error_reporter.rb +31 -0
  61. data/lib/robin/rails/notifications.rb +22 -0
  62. data/lib/robin/rails/railtie.rb +87 -0
  63. data/lib/robin/rails/transactional_push.rb +49 -0
  64. data/lib/robin/rails.rb +23 -0
  65. data/lib/robin/recovery.rb +135 -0
  66. data/lib/robin/redis/connection.rb +639 -0
  67. data/lib/robin/redis/pool.rb +190 -0
  68. data/lib/robin/redis/resp.rb +77 -0
  69. data/lib/robin/reliable_push.rb +153 -0
  70. data/lib/robin/retrier.rb +284 -0
  71. data/lib/robin/scheduler.rb +102 -0
  72. data/lib/robin/scripts.rb +331 -0
  73. data/lib/robin/statsd.rb +135 -0
  74. data/lib/robin/swarm.rb +319 -0
  75. data/lib/robin/systemd.rb +47 -0
  76. data/lib/robin/testing.rb +158 -0
  77. data/lib/robin/top.rb +178 -0
  78. data/lib/robin/unique.rb +198 -0
  79. data/lib/robin/version.rb +5 -0
  80. data/lib/robin/warmup.rb +94 -0
  81. data/lib/robin/web/api.rb +580 -0
  82. data/lib/robin/web/batch_status.rb +44 -0
  83. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  84. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  86. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  87. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  88. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  89. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  90. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  91. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  92. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  93. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  94. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  95. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  96. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  97. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  98. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  99. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  100. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  101. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  102. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  103. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  104. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  105. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  106. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  107. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  108. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  109. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  110. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  111. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  112. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  113. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  114. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  115. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  116. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  117. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  118. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  119. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  120. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  121. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  122. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  123. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  124. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  125. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  126. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  127. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  128. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  129. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  130. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  131. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  132. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  133. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  134. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  135. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  136. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  137. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  138. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  139. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  140. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  141. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  142. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  143. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  144. data/lib/robin/web/dist/index.html +19 -0
  145. data/lib/robin/web/extension.rb +193 -0
  146. data/lib/robin/web/permissions.rb +40 -0
  147. data/lib/robin/web/prometheus.rb +79 -0
  148. data/lib/robin/web/router.rb +150 -0
  149. data/lib/robin/web/static.rb +110 -0
  150. data/lib/robin/web.rb +196 -0
  151. data/lib/robin.rb +261 -0
  152. data/lib/robin_resp/robin_resp.jar +0 -0
  153. data/sig/manifest.yaml +9 -0
  154. data/sig/robin/api.rbs +442 -0
  155. data/sig/robin/batch.rbs +292 -0
  156. data/sig/robin/cancellation.rbs +25 -0
  157. data/sig/robin/cli.rbs +102 -0
  158. data/sig/robin/client.rbs +118 -0
  159. data/sig/robin/config.rbs +341 -0
  160. data/sig/robin/cron.rbs +149 -0
  161. data/sig/robin/debounce.rbs +36 -0
  162. data/sig/robin/deploy.rbs +24 -0
  163. data/sig/robin/embedded.rbs +25 -0
  164. data/sig/robin/encryption.rbs +49 -0
  165. data/sig/robin/events.rbs +10 -0
  166. data/sig/robin/expiry.rbs +17 -0
  167. data/sig/robin/fetcher.rbs +134 -0
  168. data/sig/robin/gvl.rbs +39 -0
  169. data/sig/robin/health.rbs +30 -0
  170. data/sig/robin/iterable_job.rbs +99 -0
  171. data/sig/robin/job.rbs +173 -0
  172. data/sig/robin/job_class.rbs +55 -0
  173. data/sig/robin/job_logger.rbs +56 -0
  174. data/sig/robin/launcher.rbs +191 -0
  175. data/sig/robin/leader.rbs +43 -0
  176. data/sig/robin/limiter.rbs +317 -0
  177. data/sig/robin/manager.rbs +54 -0
  178. data/sig/robin/metrics.rbs +122 -0
  179. data/sig/robin/middleware.rbs +106 -0
  180. data/sig/robin/opentelemetry.rbs +51 -0
  181. data/sig/robin/periodic.rbs +81 -0
  182. data/sig/robin/processor.rbs +107 -0
  183. data/sig/robin/profiler.rbs +60 -0
  184. data/sig/robin/progress.rbs +22 -0
  185. data/sig/robin/rails/app_loader.rbs +10 -0
  186. data/sig/robin/rails/current_attributes.rbs +44 -0
  187. data/sig/robin/rails/error_reporter.rbs +21 -0
  188. data/sig/robin/rails/notifications.rbs +9 -0
  189. data/sig/robin/rails/transactional_push.rbs +16 -0
  190. data/sig/robin/rails.rbs +8 -0
  191. data/sig/robin/recovery.rbs +29 -0
  192. data/sig/robin/redis/connection.rbs +224 -0
  193. data/sig/robin/redis/pool.rbs +89 -0
  194. data/sig/robin/redis/resp.rbs +29 -0
  195. data/sig/robin/reliable_push.rbs +77 -0
  196. data/sig/robin/retrier.rbs +72 -0
  197. data/sig/robin/scheduler.rbs +40 -0
  198. data/sig/robin/scripts.rbs +36 -0
  199. data/sig/robin/statsd.rbs +55 -0
  200. data/sig/robin/swarm.rbs +133 -0
  201. data/sig/robin/systemd.rbs +13 -0
  202. data/sig/robin/testing.rbs +80 -0
  203. data/sig/robin/top.rbs +79 -0
  204. data/sig/robin/unique.rbs +72 -0
  205. data/sig/robin/version.rbs +3 -0
  206. data/sig/robin/warmup.rbs +44 -0
  207. data/sig/robin/web/api.rbs +196 -0
  208. data/sig/robin/web/batch_status.rbs +25 -0
  209. data/sig/robin/web/extension.rbs +106 -0
  210. data/sig/robin/web/permissions.rbs +14 -0
  211. data/sig/robin/web/prometheus.rbs +33 -0
  212. data/sig/robin/web/router.rbs +53 -0
  213. data/sig/robin/web/static.rbs +42 -0
  214. data/sig/robin/web.rbs +140 -0
  215. data/sig/robin.rbs +116 -0
  216. data/sig/robin_resp.rbs +55 -0
  217. metadata +294 -0
@@ -0,0 +1,658 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Robin
6
+ # Runs a Robin server: worker threads for every capsule, the scheduler,
7
+ # and a heartbeat thread that publishes process state, flushes metrics,
8
+ # takes part in leader election and reacts to remote control signals
9
+ # sent from the web UI.
10
+ class Launcher
11
+ # How long a stopping:<identity> marker outlives the shutdown timeout:
12
+ # long enough for the process's heartbeat to expire and the leader to
13
+ # recover its jobs if it's killed before it finishes stopping.
14
+ STOPPING_MARGIN = 3600
15
+
16
+ # Seconds interrupted jobs get to wind down, after the shutdown timeout.
17
+ HARD_SHUTDOWN_GRACE = 3
18
+
19
+ # Heartbeat round trips slower than this (microseconds) get a warning.
20
+ SLOW_REDIS_US = 50_000
21
+
22
+ # Seconds without a job finishing before an idle process collects its
23
+ # garbage (see Config#idle_gc).
24
+ IDLE_GC_AFTER = 1
25
+ # GVL contention (a percentage) that, kept up for GVL_HIGH_BEATS
26
+ # heartbeats in a row, is worth a warning.
27
+ GVL_HIGH = 50
28
+ GVL_HIGH_BEATS = 3
29
+
30
+ attr_reader :config, :identity, :managers, :collector, :leader, :started_at
31
+ attr_accessor :on_stop_request
32
+
33
+ def initialize(config, embedded: false)
34
+ @config = config
35
+ @embedded = embedded
36
+ @identity = "#{Robin.hostname}:#{Process.pid}:#{SecureRandom.hex(4)}"
37
+ @collector = Metrics::Collector.new
38
+ @leader = Leader.new(config, @identity)
39
+ @scheduler = Scheduler.new(config)
40
+ @quiet = false
41
+ @done = false
42
+ @last = {}
43
+ @managers = config.capsules.values.map { |cap| Manager.new(cap, self) }
44
+ # Fetchers with acknowledgements left waiting (see #send_acks).
45
+ @acks = Thread::Queue.new
46
+ @managers.each { |m| m.fetcher.acker = @acks }
47
+ @hooks = {leader: [], beat: []}
48
+ @suspects = []
49
+ @gvl = GVL.new
50
+ @gvl_high = 0
51
+ @wake_mutex = Mutex.new
52
+ @wake = ConditionVariable.new
53
+ end
54
+
55
+ def quiet? = @quiet
56
+
57
+ # How many workers it runs, in every capsule.
58
+ def concurrency = @managers.sum(&:concurrency)
59
+
60
+ # Has capsule +name+ run +count+ workers from now on (see
61
+ # Manager#resize), growing the Redis pool to fit them. Returns false,
62
+ # changing nothing, when the process is quiet (its workers are
63
+ # stopping), or when the pool's size was set (:pool_size) and it can't
64
+ # serve them all.
65
+ def resize(count, capsule: "default")
66
+ raise ArgumentError, "a capsule needs a concurrency above 0, not #{count.inspect}" unless count.instance_of?(Integer) && count.positive?
67
+
68
+ manager = @managers.find { |m| m.capsule.name == capsule }
69
+ raise ArgumentError, "no capsule named #{capsule.inspect}" unless manager
70
+
71
+ if quiet?
72
+ config.logger.warn("not resizing capsule #{capsule.inspect} to #{count}: the process is quiet")
73
+ return false
74
+ end
75
+ before = manager.concurrency
76
+ workers = concurrency - before + count
77
+ unless config.fit_redis_pool(workers)
78
+ config.logger.warn("not resizing capsule #{capsule.inspect} to #{count}: #{workers} workers need a Redis pool of " \
79
+ "#{workers + 2} connections, but :pool_size is #{config.redis_pool.size}")
80
+ return false
81
+ end
82
+ manager.resize(count)
83
+ config.logger.info("capsule #{capsule.inspect} now runs #{count} workers (was #{before})")
84
+ Events.emit(config, "workers.resized", capsule: capsule, from: before, to: count)
85
+ true
86
+ end
87
+
88
+ def stopping? = @done
89
+
90
+ # Housekeeping hooks run on every heartbeat (:beat) or only on the
91
+ # leader (:leader). Used by cron, batches and other subsystems.
92
+ def every_beat(&block) = @hooks.fetch(:beat) << block
93
+
94
+ def every_leader_beat(&block) = @hooks.fetch(:leader) << block
95
+
96
+ def run
97
+ # All of Robin, before any of its threads start: two threads that
98
+ # reach one autoload at once can find its constant missing on JRuby
99
+ # 10.0. (The CLI and a swarm load it already. An embedded server
100
+ # didn't.)
101
+ Robin.eager_load!
102
+ @started_at = Robin.now_ms
103
+ check!
104
+ config.freeze_capsules!
105
+ # First, so a port that's taken fails the boot before anything starts.
106
+ start_health_server
107
+ Warmup.run(config) if config.jit_warmup
108
+ Leader.register(config, @leader)
109
+ # Recover before the first heartbeat registers this process's own
110
+ # in-flight set, so recovery can't mistake it for a dead one.
111
+ safely("startup recovery") { Recovery.run(config, pool: config.housekeeping_pool) }
112
+ begin
113
+ # A failing startup hook stops the boot before any work starts.
114
+ config.fire_event(:startup, reraise: true)
115
+ rescue
116
+ Leader.unregister(config)
117
+ @health&.stop
118
+ raise
119
+ end
120
+ Periodic.attach(self) unless config.periodic_blocks.empty?
121
+ @gvl.start if config.gvl_monitoring
122
+ safely("heartbeat") { beat }
123
+ @managers.each(&:start)
124
+ @scheduler.start
125
+ @heartbeat = housekeeping_thread("robin.heartbeat") { beat }
126
+ @chores = housekeeping_thread("robin.leader") { leader_chores }
127
+ @acker = Thread.new { send_acks }
128
+ @acker.name = "robin.acks"
129
+ @acker.priority = 1 # outranks worker threads, like housekeeping
130
+ Systemd.notify("READY=1") unless @embedded
131
+ self
132
+ end
133
+
134
+ # Stop fetching new jobs; running jobs finish.
135
+ def quiet
136
+ return if @quiet
137
+
138
+ @quiet = true
139
+ config.logger.info("quieting: no new jobs will be started")
140
+ @managers.each(&:quiet)
141
+ config.fire_event(:quiet, reverse: true)
142
+ end
143
+
144
+ # Graceful shutdown: quiet, wait up to config.timeout for running jobs,
145
+ # interrupt the rest and put their jobs back on the queue.
146
+ def stop
147
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + config.timeout
148
+ Systemd.notify("STOPPING=1") unless @embedded
149
+ safely("mark stopping") { mark_stopping }
150
+ quiet
151
+ @done = true
152
+ wake_up # housekeeping threads stop now
153
+ @scheduler.stop
154
+ # Interrupted jobs get one shared grace period to wind down.
155
+ @managers.each { |m| m.stop(deadline, deadline + HARD_SHUTDOWN_GRACE) }
156
+ safely("final metrics flush") { config.redis { |c| @collector.flush(config, c) } }
157
+ if config.shutdown_requeue
158
+ safely("requeue in-flight jobs") do
159
+ config.redis do |conn|
160
+ count = Recovery.drain(config, conn, inflight_key, count: false)
161
+ config.logger.info("pushed #{count} unfinished job(s) back to their queues") if count > 0
162
+ end
163
+ end
164
+ end
165
+ safely("unregister") do
166
+ config.redis do |conn|
167
+ @leader.resign(conn)
168
+ conn.pipelined do |p|
169
+ p.call("SREM", config.key("processes"), identity)
170
+ p.call("SREM", config.key("inflights"), inflight_key)
171
+ p.call("DEL", proc_key, "#{proc_key}:work", "#{proc_key}-signals", stopping_key)
172
+ end
173
+ end
174
+ end
175
+ Leader.unregister(config)
176
+ # Let a heartbeat or chore that's already running finish, so it can't
177
+ # re-register this process after it has unregistered.
178
+ @heartbeat&.join(1)
179
+ @chores&.join(1)
180
+ @acks << nil # the workers are done
181
+ @acker&.join(1)
182
+ @health&.stop
183
+ @gvl.stop
184
+ config.fire_event(:shutdown, reverse: true)
185
+ config.logger.info("bye!")
186
+ end
187
+
188
+ # This process's health, as served by Robin::HealthServer: healthy
189
+ # while its heartbeat reaches Redis and it isn't shutting down. A quiet
190
+ # process is still healthy (it's finishing its jobs).
191
+ def health
192
+ age = @beat_at && (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @beat_at)
193
+ status = if @done then "stopping"
194
+ elsif age.nil? || age > config.process_ttl then "stale"
195
+ elsif @quiet then "quiet"
196
+ else "ok"
197
+ end
198
+ {"healthy" => %w[ok quiet].include?(status), "status" => status, "identity" => identity,
199
+ "heartbeat_age" => age&.round(3), "busy" => busy, "concurrency" => concurrency}
200
+ end
201
+
202
+ def inflight_key = config.key("inflight", identity)
203
+
204
+ def proc_key = config.process_key(identity)
205
+
206
+ def stopping_key = config.key("stopping", identity)
207
+
208
+ # -- worker callbacks ----------------------------------------------------
209
+
210
+ # Records what a worker is running, for the heartbeat (and its
211
+ # in-flight key, for the sweep). Only the job's raw payload, exactly
212
+ # as fetched, is kept: the live payload hash is being changed by the
213
+ # job (and holds decrypted arguments).
214
+ # Each worker keeps the job it's running to itself (Processor#work),
215
+ # so starting and finishing jobs share no lock: with real threads
216
+ # (JRuby) a shared one would serialize every worker twice per job.
217
+ def work_started(processor, unit)
218
+ processor.work = [unit.queue, unit.raw, Robin.now_ms, unit.key]
219
+ end
220
+
221
+ def work_finished(processor, payload, elapsed_ms, ok)
222
+ processor.work = nil
223
+ processor.finished_at = monotonic
224
+ processor.finished += 1
225
+ klass = Robin.display_class(payload)
226
+ @collector.record(klass, elapsed_ms, ok)
227
+ statsd = config.statsd
228
+ statsd.distribution("jobs.perform_dist", elapsed_ms, tags: {class: klass}) if ok && statsd&.distributions?
229
+ end
230
+
231
+ def work_finished_cleanup(processor)
232
+ processor.work = nil
233
+ end
234
+
235
+ def busy = running.size
236
+
237
+ # [processor, [queue, raw, run_at, in-flight key]] for every job
238
+ # running now.
239
+ def running
240
+ @managers.flat_map(&:processors).filter_map do |processor|
241
+ work = processor.work
242
+ [processor, work] if work
243
+ end
244
+ end
245
+
246
+ def process_info
247
+ {
248
+ "identity" => identity, "hostname" => Robin.hostname, "pid" => Process.pid,
249
+ "tag" => config.tag, "labels" => config.labels, "started_at" => @started_at && Robin.epoch_seconds(@started_at),
250
+ "version" => VERSION, "ruby" => RUBY_DESCRIPTION, "env" => config.environment,
251
+ "embedded" => @embedded, "concurrency" => concurrency,
252
+ "queues" => config.capsules.values.flat_map(&:queues).uniq,
253
+ "weights" => config.capsules.values.map(&:weights),
254
+ "capsules" => @managers.to_h do |m|
255
+ c = m.capsule
256
+ [c.name, {"concurrency" => m.concurrency, "mode" => c.strict? ? "strict" : "weighted", "weights" => c.weights}]
257
+ end
258
+ }
259
+ end
260
+
261
+ # One heartbeat, in one round trip: publish state, flush metrics, renew
262
+ # leadership, run housekeeping hooks and handle remote signals.
263
+ def beat
264
+ now = Robin.now_ms
265
+ work = running.map do |processor, (queue, raw, run_at)|
266
+ [processor.thread&.name || processor.object_id, work_json(queue, raw, run_at)]
267
+ end
268
+ held = held_keys
269
+ ttl = config.process_ttl
270
+ gvl = @gvl.sample
271
+ candidate = config.leader_candidate
272
+ sweep = due?(60, :sweep)
273
+ config.housekeeping_pool.with do |conn|
274
+ rtt_start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
275
+ at = {} #: Hash[Symbol, Integer]
276
+ metrics = nil #: Metrics::drained?
277
+ replies = begin
278
+ conn.multi do |tx|
279
+ tx.call("EXISTS", proc_key)
280
+ tx.call("SADD", config.key("processes"), identity)
281
+ tx.call("SADD", config.key("inflights"), inflight_key)
282
+ # Recovery unregisters the in-flight keys of a process whose
283
+ # heartbeat lapsed; they're registered again as soon as it's back.
284
+ tx.call("SADD", inflight_key, *held) unless held.empty?
285
+ # The round trip time is the last heartbeat's: this one's is
286
+ # only known once it's back.
287
+ tx.call("HSET", proc_key, "info", Robin.dump_json(process_info), "beat", Robin.epoch_seconds(now),
288
+ "busy", work.size, "concurrency", concurrency, "quiet", @quiet, "rss", rss_kb,
289
+ # (Steep doesn't take splatting nil, which adds no arguments.)
290
+ "leader", @leader.leader? ? "1" : "0", *(["rtt_us", @rtt] if @rtt), *(["gvl", gvl] if gvl)) # steep:ignore ArgumentTypeMismatch
291
+ tx.call("EXPIRE", proc_key, ttl)
292
+ tx.call("DEL", "#{proc_key}:work")
293
+ tx.call("HSET", "#{proc_key}:work", *work.flatten) unless work.empty?
294
+ tx.call("EXPIRE", "#{proc_key}:work", ttl)
295
+ at[:signals] = tx.commands.size
296
+ tx.call("LRANGE", "#{proc_key}-signals", 0, -1)
297
+ tx.call("DEL", "#{proc_key}-signals")
298
+ tx.call(*Cancellation.listing(config))
299
+ tx.call("SMEMBERS", config.key("paused"))
300
+ tx.call(*JobClass.listing(config))
301
+ at[:lease] = tx.commands.size
302
+ @leader.lease(tx) if candidate
303
+ at[:sweep] = tx.commands.size
304
+ tx.call("SMEMBERS", inflight_key) if sweep
305
+ metrics = @collector.queue(config, tx)
306
+ end
307
+ rescue
308
+ @collector.restore(metrics) if metrics
309
+ raise
310
+ end
311
+ # (The transaction went through, so metrics is set.)
312
+ @collector.flushed(config, metrics) # steep:ignore ArgumentTypeMismatch
313
+ warn_about_lapse if @beaten && replies.first.zero?
314
+ @beaten = true
315
+ @beat_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
316
+ signals, _, cancelled, paused, paused_classes = replies.drop(at.fetch(:signals)) #: [Array[String], Integer, Array[String], Array[String], Array[String]]
317
+ @rtt = Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) - rtt_start
318
+ warn_about_slow_redis(@rtt) if @rtt > SLOW_REDIS_US
319
+ check_gvl(gvl) if gvl
320
+
321
+ @leader.settle(replies.fetch(at.fetch(:lease))) if candidate
322
+ Cancellation.store(config, cancelled)
323
+ JobClass.store(config, paused_classes)
324
+ hand_back_stale(conn, paused)
325
+ @hooks.fetch(:beat).each { |h| safely("beat hook") { h.call(conn) } }
326
+ safely("sweep") { sweep_inflight(conn, replies.fetch(at.fetch(:sweep))) } if sweep
327
+ signals.reverse_each { |sig| handle_signal(sig) } # pushed onto the head: oldest last
328
+ end
329
+ check_memory
330
+ unless @embedded
331
+ collect_idle_garbage
332
+ Process.setproctitle(title)
333
+ Systemd.notify("WATCHDOG=1") if Systemd.watchdog?
334
+ end
335
+ config.fire_event(:heartbeat)
336
+ end
337
+
338
+ # Runs a full GC once the process has gone idle (see Config#idle_gc):
339
+ # no job running, none finished for IDLE_GC_AFTER seconds, and at
340
+ # least config.idle_gc finished since the last time. Each worker
341
+ # counts its own jobs (Processor#finished), so finishing one takes no
342
+ # lock; the counts at the last collection are kept by worker.
343
+ def collect_idle_garbage
344
+ least = config.idle_gc
345
+ return unless least && running.empty?
346
+
347
+ processors = @managers.flat_map(&:processors)
348
+ last = processors.filter_map(&:finished_at).max
349
+ return unless last && monotonic - last >= IDLE_GC_AFTER
350
+
351
+ counted = @collected_counts || {}
352
+ return if processors.sum { |p| p.finished - counted.fetch(p, 0) } < least
353
+
354
+ collect_garbage
355
+ @collected_counts = processors.to_h { |p| [p, p.finished] }
356
+ end
357
+
358
+ def collect_garbage = GC.start
359
+
360
+ # Puts back the jobs every capsule fetched for its workers that are
361
+ # from a queue paused since, or that no worker took in time (see
362
+ # Fetcher#stale), in one round trip.
363
+ def hand_back_stale(conn, paused)
364
+ stale = @managers.map { |m| [m.fetcher, m.fetcher.stale(paused)] } #: Array[[Fetcher, Array[Fetcher::UnitOfWork]]]
365
+ stale.reject! { |_fetcher, units| units.empty? }
366
+ return if stale.empty?
367
+
368
+ conn.multi { |tx| stale.each { |fetcher, units| fetcher.hand_back(tx, units) } }
369
+ stale.each { |fetcher, units| fetcher.freed(units) }
370
+ end
371
+
372
+ # The process title (as ps shows it) of a standalone server.
373
+ def title
374
+ state = if @done then " stopping"
375
+ elsif @quiet then " quiet"
376
+ end
377
+ "robin #{VERSION} #{config.tag} [#{busy} of #{concurrency} busy]#{state}"
378
+ end
379
+
380
+ # Remote signals are named like the Unix signals that do the same.
381
+ def handle_signal(signal)
382
+ case signal
383
+ when "TSTP" then quiet
384
+ when "TERM" then request_stop
385
+ when "TTIN" then dump_threads
386
+ # (The match sets $1 and $2.)
387
+ when /\ARESIZE (\d+) (.+)\z/m then safely("resize") { resize(Integer($1, 10), capsule: $2) } # steep:ignore ArgumentTypeMismatch
388
+ else config.logger.warn("ignoring unknown remote signal #{signal.inspect}")
389
+ end
390
+ end
391
+
392
+ def request_stop
393
+ if on_stop_request
394
+ on_stop_request.call
395
+ else
396
+ Thread.new { stop }
397
+ end
398
+ end
399
+
400
+ def dump_threads
401
+ Thread.list.each do |t|
402
+ config.logger.warn("Thread #{t.name || t.object_id} (#{t.status})")
403
+ config.logger.warn((t.backtrace || ["<no backtrace>"]).join("\n"))
404
+ end
405
+ end
406
+
407
+ private
408
+
409
+ # {"queue", "run_at" (epoch seconds), "payload" (the job's JSON, as
410
+ # fetched)} as JSON.
411
+ def work_json(queue, raw, run_at)
412
+ Robin.dump_json({"queue" => queue, "payload" => raw, "run_at" => Robin.epoch_seconds(run_at)})
413
+ end
414
+
415
+ MINIMUM_REDIS = [6, 2].freeze
416
+
417
+ # Refuses settings and servers Robin can't run correctly with, and
418
+ # warns about ones that risk losing data.
419
+ def check!
420
+ config.capsules.each_value do |capsule|
421
+ raise ArgumentError, "capsule #{capsule.name.inspect} needs a concurrency above 0" unless capsule.concurrency.positive?
422
+ end
423
+ raise ArgumentError, "the shutdown timeout must be above 0 seconds" unless config.timeout.positive?
424
+ raise ArgumentError, "the fetch timeout must be above 0 seconds" unless config.fetch_timeout.positive?
425
+
426
+ needed = config.total_concurrency + 2
427
+ if config.redis_pool.size < needed
428
+ raise ArgumentError, "the Redis pool has #{config.redis_pool.size} connections but #{config.total_concurrency} " \
429
+ "worker threads need at least #{needed}; raise :pool_size or leave it unset"
430
+ end
431
+
432
+ info, conn = config.redis { |c| [c.call("INFO"), c] }
433
+ version = info[/^redis_version:(\S+)/, 1]
434
+ if (version.split(".").map(&:to_i) <=> MINIMUM_REDIS).negative?
435
+ raise ArgumentError, "Robin needs Redis #{MINIMUM_REDIS.join(".")} or later; this server runs #{version}"
436
+ end
437
+ config.logger.info(connected(conn, version))
438
+
439
+ policy = info[/^maxmemory_policy:(\S+)/, 1]
440
+ return if policy.nil? || policy == "noeviction"
441
+
442
+ config.logger.warn("Redis maxmemory-policy is #{policy}; set it to noeviction, or Redis may evict " \
443
+ "jobs and process heartbeats when memory runs low")
444
+ end
445
+
446
+ # Where this process's jobs are, so connecting to the wrong Redis (the
447
+ # default 127.0.0.1:6379, say) is plain to see: the server (never its
448
+ # password) and the key prefixes.
449
+ def connected(conn, version)
450
+ tls = " over TLS" if conn.options.fetch(:ssl)
451
+ own = config.internal_key_prefix
452
+ own = (", Robin's own under #{own.inspect}" unless own == config.key_prefix)
453
+ "Redis #{version} at #{conn.id}#{tls}; keys under #{config.key_prefix.inspect}#{own}"
454
+ end
455
+
456
+ # Swarm children listen on the configured port plus their index.
457
+ def start_health_server
458
+ return unless config.health_port
459
+
460
+ port = config.health_port + Integer(ENV.fetch("ROBIN_SWARM_INDEX", 0))
461
+ @health = HealthServer.new(self, port: port, bind: config.health_bind)
462
+ config.logger.info("health check listening on #{config.health_bind}:#{@health.port}")
463
+ end
464
+
465
+ # Jobs interrupted by this shutdown aren't poison pills, even if the
466
+ # process is killed before it can requeue them itself.
467
+ def mark_stopping
468
+ config.redis { |conn| conn.call("SET", stopping_key, "1", "EX", (config.timeout + STOPPING_MARGIN).ceil) }
469
+ end
470
+
471
+ # A thread that runs the block every heartbeat interval until stopping.
472
+ # It outranks worker threads, so busy jobs can't starve it.
473
+ def housekeeping_thread(name)
474
+ Thread.new do
475
+ Thread.current.name = name
476
+ Thread.current.priority = 1
477
+ until @done
478
+ nap(config.heartbeat_interval)
479
+ safely(name.delete_prefix("robin.")) { yield } unless @done
480
+ end
481
+ end
482
+ end
483
+
484
+ # Sleeps up to +seconds+, or until #wake_up (called by #stop).
485
+ # mutant:disable -- the lock and the @done check only close a lost-wakeup
486
+ # race (a stop landing between the loop's check and the wait) that tests
487
+ # can't provoke deterministically; test_stop_wakes_sleeping_housekeeping_threads
488
+ # covers the behavior.
489
+ def nap(seconds)
490
+ @wake_mutex.synchronize { @wake.wait(@wake_mutex, seconds) unless @done }
491
+ end
492
+
493
+ # mutant:disable -- see #nap
494
+ def wake_up
495
+ @wake_mutex.synchronize { @wake.broadcast }
496
+ end
497
+
498
+ # Sends the acknowledgements fetchers leave waiting, ACK_WAIT after
499
+ # they put themselves on @acks (see Fetcher#acker), until a nil comes
500
+ # off it. Only the first failure of a run of them is reported (workers
501
+ # report a Redis outage too); those acknowledgements wait again.
502
+ # mutant:disable -- Thread::Queue#shift is an alias of #pop.
503
+ def send_acks
504
+ failing = false
505
+ loop do
506
+ fetchers = [@acks.pop]
507
+ break unless fetchers.first
508
+
509
+ sleep Fetcher::ACK_WAIT
510
+ fetchers << @acks.pop until @acks.empty? # this thread is the only taker
511
+ fetchers.compact.each do |fetcher|
512
+ fetcher.send_acks
513
+ failing = false
514
+ rescue => e
515
+ config.handle_exception(e, {context: "acknowledging jobs"}) unless failing
516
+ failing = true
517
+ end
518
+ break if fetchers.include?(nil)
519
+ end
520
+ end
521
+
522
+ # Cluster-wide duties of the leader, on their own thread so slow ones
523
+ # (recovery after a big crash) never delay this process's heartbeat.
524
+ def leader_chores
525
+ return unless @leader.leader?
526
+
527
+ config.housekeeping_pool.with do |conn|
528
+ every(60, :sample_queues) { Metrics.sample_queues(config, conn) }
529
+ every(30, :recovery) { Recovery.run(config, pool: config.housekeeping_pool) }
530
+ every(60, :prune) { prune_processes(conn) }
531
+ @hooks.fetch(:leader).each { |h| safely("leader hook") { h.call(conn) } }
532
+ end
533
+ end
534
+
535
+ # The heartbeat key had expired: for a while this process looked dead, so
536
+ # the leader may have recovered (and another process re-run) jobs that
537
+ # were still running here.
538
+ # At most every 10 minutes: a slow Redis slows every job down.
539
+ def warn_about_slow_redis(rtt_us)
540
+ every(600, :slow_redis) do
541
+ config.logger.warn(format("Redis round trips take %.1fms; jobs will be slow (is Redis far away, or overloaded?)", rtt_us / 1000.0))
542
+ Events.emit(config, "redis.slow", rtt_us: rtt_us)
543
+ end
544
+ end
545
+
546
+ # Contention is reported every heartbeat (StatsD's gvl.contention);
547
+ # contention kept high warns, at most every 10 minutes.
548
+ def check_gvl(percent)
549
+ config.statsd&.gauge("gvl.contention", percent)
550
+ @gvl_high = (percent >= GVL_HIGH) ? @gvl_high + 1 : 0
551
+ return if @gvl_high < GVL_HIGH_BEATS
552
+
553
+ every(600, :gvl) do
554
+ config.logger.warn("GVL contention is #{percent}%: threads spend most of their time waiting for each other; " \
555
+ "fewer threads per process (and more processes) would run CPU-heavy jobs faster")
556
+ Events.emit(config, "gvl.contended", percent: percent)
557
+ end
558
+ end
559
+
560
+ def warn_about_lapse
561
+ config.logger.warn("heartbeat lapsed (more than #{config.process_ttl}s without one); jobs running here " \
562
+ "may have been recovered and run again elsewhere. Is the process starved of CPU or Redis connections?")
563
+ Events.emit(config, "heartbeat.lapsed")
564
+ end
565
+
566
+ # The in-flight keys this process's workers hold: those of running
567
+ # jobs, of finished jobs whose acknowledgement waits for a later
568
+ # fetch, of jobs fetched for a worker that hasn't asked yet, and free
569
+ # ones. Running jobs are read first, so one finishing meanwhile is seen
570
+ # either way.
571
+ def held_keys
572
+ held = running.map { |_processor, work| work.fetch(3) } #: Array[String?]
573
+ held.concat(@managers.flat_map { |m| m.processors.map(&:unacked) + m.fetcher.held })
574
+ held.compact
575
+ end
576
+
577
+ # Puts back jobs that sit in this process's in-flight keys without any
578
+ # worker holding them (e.g. one died after the job finished but before
579
+ # acknowledging it, or a round trip failed after filling them), and
580
+ # unregisters keys left empty that way. A key must be unheld, with the
581
+ # same job (or none) in it, on two sweeps in a row, so one that's just
582
+ # being fetched or acknowledged (or was used again since) is never
583
+ # taken.
584
+ # +registered+ are the keys registered for this process, as the
585
+ # heartbeat read them.
586
+ def sweep_inflight(conn, registered)
587
+ unheld = registered - held_keys
588
+ sightings = unheld.zip(conn.pipelined { |p| unheld.each { |key| p.call("LINDEX", key, 0) } })
589
+ orphans = sightings & @suspects
590
+ @suspects = sightings - orphans
591
+ orphans.select! { |key, _raw| @managers.all? { |m| m.fetcher.forget(key) } }
592
+ requeued = conn.pipelined do |p|
593
+ orphans.each do |key, raw|
594
+ # An empty key (raw nil) is only unregistered.
595
+ queue = raw ? Robin.load_json(raw).fetch("queue") : ""
596
+ p.evalsha(Scripts::REQUEUE_ONE, [key, config.key("queue", queue), config.key("queues"), inflight_key], [raw.to_s, queue])
597
+ end
598
+ end.sum
599
+ return unless requeued.positive?
600
+
601
+ config.logger.warn("put back #{requeued} orphaned in-flight job(s)")
602
+ config.statsd&.increment("jobs.orphaned", requeued)
603
+ Events.emit(config, "jobs.orphaned", count: requeued)
604
+ end
605
+
606
+ def every(seconds, name)
607
+ safely(name.to_s) { yield } if due?(seconds, name)
608
+ end
609
+
610
+ # Whether +seconds+ have passed since +name+ was last due (or it never
611
+ # was); it's due again +seconds+ from now if so.
612
+ def due?(seconds, name)
613
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
614
+ last = @last[name]
615
+ return false if last && now - last < seconds
616
+
617
+ @last[name] = now
618
+ end
619
+
620
+ # mutant:disable -- Connection#evalsha splats ARGV, so passing the
621
+ # prefix bare or wrapped in an Array is identical.
622
+ def prune_processes(conn) = conn.evalsha(Scripts::PRUNE, [config.key("processes")], [config.key_prefix])
623
+
624
+ def check_memory
625
+ limit = config.memory_limit_mb
626
+ return unless limit && !@done
627
+
628
+ rss_mb = rss_kb / 1024
629
+ return if rss_mb < limit
630
+
631
+ config.logger.warn("RSS #{rss_mb}MB exceeds memory_limit_mb #{limit}MB; restarting gracefully")
632
+ Events.emit(config, "memory.exceeded", rss_mb: rss_mb, limit_mb: limit)
633
+ request_stop
634
+ end
635
+
636
+ def rss_kb
637
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
638
+ return @rss if @rss && now - @rss_at < 15
639
+
640
+ @rss_at = now
641
+ @rss = if File.readable?("/proc/self/status")
642
+ Integer(File.read("/proc/self/status")[/^VmRSS:\s*(\d+)/, 1])
643
+ else
644
+ Integer(`ps -o rss= -p #{Process.pid}`)
645
+ end
646
+ rescue
647
+ @rss = 0
648
+ end
649
+
650
+ def safely(what)
651
+ yield
652
+ rescue => e
653
+ config.handle_exception(e, {context: what})
654
+ end
655
+
656
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
657
+ end
658
+ end