robin 0.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/java/robin_resp/Blocking.java +27 -0
  13. data/ext/java/robin_resp/NativeIO.java +229 -0
  14. data/ext/java/robin_resp/NativeRESP.java +279 -0
  15. data/ext/java/robin_resp/NativeSlots.java +133 -0
  16. data/ext/java/robin_resp/RobinRespService.java +32 -0
  17. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  18. data/lib/generators/robin/install/install_generator.rb +39 -0
  19. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  20. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  21. data/lib/generators/robin/job/job_generator.rb +78 -0
  22. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  23. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  24. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  25. data/lib/puma/plugin/robin.rb +131 -0
  26. data/lib/robin/active_job.rb +144 -0
  27. data/lib/robin/api.rb +777 -0
  28. data/lib/robin/batch.rb +700 -0
  29. data/lib/robin/cancellation.rb +62 -0
  30. data/lib/robin/cli.rb +325 -0
  31. data/lib/robin/client.rb +372 -0
  32. data/lib/robin/config.rb +461 -0
  33. data/lib/robin/cron.rb +325 -0
  34. data/lib/robin/debounce.rb +108 -0
  35. data/lib/robin/deploy.rb +65 -0
  36. data/lib/robin/embedded.rb +53 -0
  37. data/lib/robin/encryption.rb +147 -0
  38. data/lib/robin/events.rb +60 -0
  39. data/lib/robin/expiry.rb +41 -0
  40. data/lib/robin/fetcher.rb +441 -0
  41. data/lib/robin/gvl.rb +62 -0
  42. data/lib/robin/health.rb +76 -0
  43. data/lib/robin/iterable_job.rb +294 -0
  44. data/lib/robin/job.rb +258 -0
  45. data/lib/robin/job_class.rb +157 -0
  46. data/lib/robin/job_logger.rb +121 -0
  47. data/lib/robin/launcher.rb +658 -0
  48. data/lib/robin/leader.rb +86 -0
  49. data/lib/robin/limiter.rb +639 -0
  50. data/lib/robin/manager.rb +125 -0
  51. data/lib/robin/metrics.rb +303 -0
  52. data/lib/robin/middleware.rb +165 -0
  53. data/lib/robin/opentelemetry.rb +88 -0
  54. data/lib/robin/periodic.rb +327 -0
  55. data/lib/robin/processor.rb +317 -0
  56. data/lib/robin/profiler.rb +140 -0
  57. data/lib/robin/progress.rb +46 -0
  58. data/lib/robin/rails/app_loader.rb +23 -0
  59. data/lib/robin/rails/current_attributes.rb +88 -0
  60. data/lib/robin/rails/error_reporter.rb +31 -0
  61. data/lib/robin/rails/notifications.rb +22 -0
  62. data/lib/robin/rails/railtie.rb +87 -0
  63. data/lib/robin/rails/transactional_push.rb +49 -0
  64. data/lib/robin/rails.rb +23 -0
  65. data/lib/robin/recovery.rb +135 -0
  66. data/lib/robin/redis/connection.rb +639 -0
  67. data/lib/robin/redis/pool.rb +190 -0
  68. data/lib/robin/redis/resp.rb +77 -0
  69. data/lib/robin/reliable_push.rb +153 -0
  70. data/lib/robin/retrier.rb +284 -0
  71. data/lib/robin/scheduler.rb +102 -0
  72. data/lib/robin/scripts.rb +331 -0
  73. data/lib/robin/statsd.rb +135 -0
  74. data/lib/robin/swarm.rb +319 -0
  75. data/lib/robin/systemd.rb +47 -0
  76. data/lib/robin/testing.rb +158 -0
  77. data/lib/robin/top.rb +178 -0
  78. data/lib/robin/unique.rb +198 -0
  79. data/lib/robin/version.rb +5 -0
  80. data/lib/robin/warmup.rb +94 -0
  81. data/lib/robin/web/api.rb +580 -0
  82. data/lib/robin/web/batch_status.rb +44 -0
  83. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  84. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  86. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  87. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  88. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  89. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  90. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  91. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  92. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  93. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  94. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  95. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  96. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  97. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  98. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  99. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  100. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  101. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  102. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  103. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  104. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  105. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  106. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  107. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  108. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  109. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  110. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  111. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  112. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  113. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  114. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  115. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  116. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  117. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  118. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  119. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  120. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  121. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  122. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  123. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  124. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  125. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  126. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  127. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  128. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  129. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  130. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  131. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  132. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  133. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  134. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  135. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  136. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  137. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  138. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  139. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  140. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  141. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  142. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  143. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  144. data/lib/robin/web/dist/index.html +19 -0
  145. data/lib/robin/web/extension.rb +193 -0
  146. data/lib/robin/web/permissions.rb +40 -0
  147. data/lib/robin/web/prometheus.rb +79 -0
  148. data/lib/robin/web/router.rb +150 -0
  149. data/lib/robin/web/static.rb +110 -0
  150. data/lib/robin/web.rb +196 -0
  151. data/lib/robin.rb +261 -0
  152. data/lib/robin_resp/robin_resp.jar +0 -0
  153. data/sig/manifest.yaml +9 -0
  154. data/sig/robin/api.rbs +442 -0
  155. data/sig/robin/batch.rbs +292 -0
  156. data/sig/robin/cancellation.rbs +25 -0
  157. data/sig/robin/cli.rbs +102 -0
  158. data/sig/robin/client.rbs +118 -0
  159. data/sig/robin/config.rbs +341 -0
  160. data/sig/robin/cron.rbs +149 -0
  161. data/sig/robin/debounce.rbs +36 -0
  162. data/sig/robin/deploy.rbs +24 -0
  163. data/sig/robin/embedded.rbs +25 -0
  164. data/sig/robin/encryption.rbs +49 -0
  165. data/sig/robin/events.rbs +10 -0
  166. data/sig/robin/expiry.rbs +17 -0
  167. data/sig/robin/fetcher.rbs +134 -0
  168. data/sig/robin/gvl.rbs +39 -0
  169. data/sig/robin/health.rbs +30 -0
  170. data/sig/robin/iterable_job.rbs +99 -0
  171. data/sig/robin/job.rbs +173 -0
  172. data/sig/robin/job_class.rbs +55 -0
  173. data/sig/robin/job_logger.rbs +56 -0
  174. data/sig/robin/launcher.rbs +191 -0
  175. data/sig/robin/leader.rbs +43 -0
  176. data/sig/robin/limiter.rbs +317 -0
  177. data/sig/robin/manager.rbs +54 -0
  178. data/sig/robin/metrics.rbs +122 -0
  179. data/sig/robin/middleware.rbs +106 -0
  180. data/sig/robin/opentelemetry.rbs +51 -0
  181. data/sig/robin/periodic.rbs +81 -0
  182. data/sig/robin/processor.rbs +107 -0
  183. data/sig/robin/profiler.rbs +60 -0
  184. data/sig/robin/progress.rbs +22 -0
  185. data/sig/robin/rails/app_loader.rbs +10 -0
  186. data/sig/robin/rails/current_attributes.rbs +44 -0
  187. data/sig/robin/rails/error_reporter.rbs +21 -0
  188. data/sig/robin/rails/notifications.rbs +9 -0
  189. data/sig/robin/rails/transactional_push.rbs +16 -0
  190. data/sig/robin/rails.rbs +8 -0
  191. data/sig/robin/recovery.rbs +29 -0
  192. data/sig/robin/redis/connection.rbs +224 -0
  193. data/sig/robin/redis/pool.rbs +89 -0
  194. data/sig/robin/redis/resp.rbs +29 -0
  195. data/sig/robin/reliable_push.rbs +77 -0
  196. data/sig/robin/retrier.rbs +72 -0
  197. data/sig/robin/scheduler.rbs +40 -0
  198. data/sig/robin/scripts.rbs +36 -0
  199. data/sig/robin/statsd.rbs +55 -0
  200. data/sig/robin/swarm.rbs +133 -0
  201. data/sig/robin/systemd.rbs +13 -0
  202. data/sig/robin/testing.rbs +80 -0
  203. data/sig/robin/top.rbs +79 -0
  204. data/sig/robin/unique.rbs +72 -0
  205. data/sig/robin/version.rbs +3 -0
  206. data/sig/robin/warmup.rbs +44 -0
  207. data/sig/robin/web/api.rbs +196 -0
  208. data/sig/robin/web/batch_status.rbs +25 -0
  209. data/sig/robin/web/extension.rbs +106 -0
  210. data/sig/robin/web/permissions.rbs +14 -0
  211. data/sig/robin/web/prometheus.rbs +33 -0
  212. data/sig/robin/web/router.rbs +53 -0
  213. data/sig/robin/web/static.rbs +42 -0
  214. data/sig/robin/web.rbs +140 -0
  215. data/sig/robin.rbs +116 -0
  216. data/sig/robin_resp.rbs +55 -0
  217. metadata +294 -0
@@ -0,0 +1,319 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "etc"
4
+ require "rbconfig"
5
+ require "socket"
6
+
7
+ module Robin
8
+ # Runs several Robin processes on one machine from a single command.
9
+ #
10
+ # robin swarm -n 4 -r ./config/environment.rb -c 10
11
+ #
12
+ # Where fork is available, the parent boots the application once and
13
+ # forks children, so they share memory copy-on-write. Elsewhere (JRuby)
14
+ # each child is a fresh `robin` process started with the same options.
15
+ # Dead children are replaced, with a growing delay if they keep crashing.
16
+ # Signals sent to the parent:
17
+ #
18
+ # TERM, INT:: graceful shutdown of every child, then exit
19
+ # TSTP:: quiet every child (finish running jobs, fetch no more)
20
+ # TTIN:: every child logs its thread backtraces
21
+ # USR2:: rolling restart: each child in turn is replaced by a fresh
22
+ # one, which must be up (heartbeating) *before* the old one
23
+ # stops, so capacity never drops
24
+ #
25
+ # The number of children comes from -n, else ROBIN_SWARM_COUNT, else the
26
+ # number of CPU cores. A decimal means that many per core: 0.5 on 8 cores
27
+ # is 4 children, 1.5 is 12 (always at least 1).
28
+ #
29
+ # Children see ROBIN_SWARM_INDEX (0...n) and stop by themselves if the
30
+ # parent dies. A child exceeding config.memory_limit_mb stops gracefully
31
+ # and is replaced.
32
+ class Swarm
33
+ READY_TIMEOUT = 60
34
+ MIN_UPTIME = 10
35
+ MAX_DELAY = 60
36
+ SIGNALS = %w[INT TERM TSTP TTIN USR2 CHLD].freeze
37
+ EXE = File.expand_path("../../exe/robin", __dir__)
38
+ LIB = File.expand_path("..", __dir__)
39
+
40
+ Child = Struct.new(:pid, :index, :started_at, :retiring)
41
+
42
+ # Operating-system primitives, separated so the supervisor logic can
43
+ # be tested without real processes.
44
+ class OS
45
+ SIGNAL_BUFFER = 1024
46
+
47
+ # mutant:disable -- only differs where fork is missing (JRuby);
48
+ # mutant runs on MRI.
49
+ def fork? = Process.respond_to?(:fork)
50
+
51
+ # mutant:disable -- `super` reaches Kernel#fork, the same call.
52
+ def fork(&) = Process.fork(&)
53
+
54
+ # mutant:disable -- `super` reaches Kernel#spawn, the same call.
55
+ def spawn(env, *command) = Process.spawn(env, *command)
56
+
57
+ # Tidies the heap (GC, compaction) so forked children share more
58
+ # memory copy-on-write.
59
+ def warmup = Process.warmup
60
+
61
+ def kill(pid, signal)
62
+ Process.kill(signal, pid)
63
+ rescue Errno::ESRCH
64
+ nil
65
+ end
66
+
67
+ # [pid, status] of an exited child, or nil.
68
+ def wait(block: false)
69
+ wait2(block)
70
+ rescue Errno::ECHILD
71
+ nil
72
+ end
73
+
74
+ def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
75
+
76
+ # mutant:disable -- `super` reaches Kernel#sleep, the same call.
77
+ def sleep(seconds) = Kernel.sleep(seconds)
78
+
79
+ # Routes +signals+ into a pipe; returns its reading end.
80
+ def trap_signals(signals)
81
+ reader, writer = IO.pipe
82
+ signals.each { |sig| trap(sig) { notify(writer, sig) } }
83
+ reader
84
+ end
85
+
86
+ def reset_signals(signals) = signals.each { |sig| trap(sig, "DEFAULT") }
87
+
88
+ # Signals received within +timeout+ seconds.
89
+ def read_signals(reader, timeout)
90
+ return [] unless reader.wait_readable(timeout)
91
+
92
+ reader.read_nonblock(SIGNAL_BUFFER).lines(chomp: true)
93
+ end
94
+
95
+ # mutant:disable -- `self.exit!` reaches Kernel#exit!, the same call.
96
+ def finish_child = Kernel.exit!(0)
97
+
98
+ private
99
+
100
+ # mutant:disable -- equivalent spellings: wait2's pid -1 or omitted,
101
+ # flags 0 or nil.
102
+ def wait2(block) = Process.wait2(-1, block ? 0 : Process::WNOHANG)
103
+
104
+ # mutant:disable -- raising instead only differs once 64KB of
105
+ # unread signal names fill the pipe.
106
+ def notify(writer, signal) = writer.write_nonblock("#{signal}\n", exception: false)
107
+ end
108
+
109
+ # Stops this process when +parent+ goes away. Children call this; with
110
+ # spawn they get here by loading this file (see #child_command).
111
+ def self.watch_parent(parent, os: OS.new, interval: 1)
112
+ Thread.new do
113
+ Thread.current.name = "robin.swarm.watch"
114
+ os.sleep(interval) while Process.ppid == parent
115
+ os.kill(Process.pid, "TERM")
116
+ end
117
+ end
118
+
119
+ # Watches the swarm parent named in ROBIN_SWARM_PARENT, if it is ours.
120
+ # Children for a count setting (see the class comment).
121
+ def self.process_count(value, cores = Etc.nprocessors)
122
+ return cores if value.nil?
123
+
124
+ text = value.to_s
125
+ number = text.include?(".") ? Float(text) : Integer(text)
126
+ raise ArgumentError, "the swarm's process count must be positive: #{text}" unless number.positive?
127
+
128
+ number.integer? ? number : [(number * cores).round, 1].max #: Integer
129
+ end
130
+
131
+ def self.watch_parent_from_env(env = ENV, os: OS.new)
132
+ parent = env["ROBIN_SWARM_PARENT"]
133
+ watch_parent(Process.ppid, os: os) if parent && Integer(parent) == Process.ppid
134
+ end
135
+
136
+ attr_reader :children, :count, :mode
137
+
138
+ def initialize(cli, os: OS.new)
139
+ @cli = cli
140
+ @config = cli.config
141
+ @os = os
142
+ @count = Swarm.process_count(cli.options[:processes] || ENV["ROBIN_SWARM_COUNT"])
143
+ @mode = ENV["ROBIN_SWARM_MODE"] || (os.fork? ? "fork" : "spawn")
144
+ @children = {}
145
+ @crashes = Hash.new(0)
146
+ @restarts = {} # index => when (monotonic) to start a replacement
147
+ end
148
+
149
+ def run
150
+ @cli.boot
151
+ if @mode == "fork"
152
+ # Loaded before forking, the code is shared copy-on-write instead
153
+ # of autoloaded by each child.
154
+ Robin.eager_load!
155
+ @os.warmup
156
+ end
157
+ @signals = @os.trap_signals(SIGNALS)
158
+ logger.info("swarm #{Process.pid} starting #{@count} children (#{@mode})")
159
+ @count.times { |i| start_child(i) }
160
+ until @stopping # shutdown returns once every child is gone
161
+ reap
162
+ start_due_restarts
163
+ @os.read_signals(@signals, 1).each { |sig| handle(sig) }
164
+ end
165
+ logger.info("swarm #{Process.pid} stopped")
166
+ end
167
+
168
+ def handle(signal)
169
+ case signal
170
+ when "INT", "TERM" then shutdown
171
+ when "TSTP", "TTIN" then signal_children(signal)
172
+ when "USR2" then rolling_restart
173
+ end
174
+ end
175
+
176
+ def shutdown
177
+ return if @stopping
178
+
179
+ @stopping = true
180
+ @restarts.clear
181
+ logger.info("swarm stopping #{@children.size} children")
182
+ signal_children("TERM")
183
+ return if wait_for { @children.empty? }
184
+
185
+ logger.warn("killing #{@children.size} children that didn't stop in time")
186
+ signal_children("KILL")
187
+ reap(block: true) until @children.empty?
188
+ end
189
+
190
+ def rolling_restart
191
+ return if @stopping
192
+
193
+ logger.info("swarm rolling restart")
194
+ @children.values.sort_by(&:index).each do |old|
195
+ next unless @children.key?(old.pid)
196
+
197
+ old.retiring = true
198
+ replacement = start_child(old.index)
199
+ if ready?(replacement)
200
+ stop_child(old)
201
+ else
202
+ logger.error("replacement for child #{old.index} didn't start; keeping the old one")
203
+ replacement.retiring = true
204
+ @os.kill(replacement.pid, "KILL")
205
+ old.retiring = false
206
+ end
207
+ end
208
+ end
209
+
210
+ # The options a spawned child needs to run like this process.
211
+ def child_arguments
212
+ o = @cli.options
213
+ args = [] #: Array[String]
214
+ {require: "-r", config_file: "-C", environment: "-e", concurrency: "-c", timeout: "-t", tag: "-g"}.each do |key, flag|
215
+ args.push(flag, o.fetch(key).to_s) if o[key]
216
+ end
217
+ Array(o[:queues]).each { |q| args.push("-q", [q].join(",")) }
218
+ args << "-v" if o[:verbose]
219
+ args << "--json-logs" if o[:json_logs]
220
+ args
221
+ end
222
+
223
+ def child_command = [RbConfig.ruby, "-r", "robin/swarm", EXE, *child_arguments]
224
+
225
+ def child_env(index)
226
+ {"ROBIN_SWARM_INDEX" => index.to_s, "ROBIN_SWARM_PARENT" => Process.pid.to_s,
227
+ "RUBYLIB" => [LIB, ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR)}
228
+ end
229
+
230
+ def start_child(index)
231
+ pid = (@mode == "fork") ? @os.fork { child_main(index) } : @os.spawn(child_env(index), *child_command)
232
+ child = Child.new(pid, index, @os.now, false)
233
+ @children[pid] = child
234
+ logger.info("swarm child #{index} started (pid #{pid})")
235
+ child
236
+ end
237
+
238
+ # Runs inside a forked child.
239
+ def child_main(index)
240
+ @os.reset_signals(SIGNALS)
241
+ ENV["ROBIN_SWARM_INDEX"] = index.to_s
242
+ Swarm.watch_parent(Process.ppid, os: @os)
243
+ @cli.config.fire_event(:fork)
244
+ @cli.serve
245
+ @os.finish_child
246
+ end
247
+
248
+ # Collects exited children and schedules replacements.
249
+ def reap(block: false)
250
+ loop do
251
+ waited = @os.wait(block: block) or break
252
+ pid, status = waited
253
+
254
+ child = @children.delete(pid)
255
+ next unless child
256
+
257
+ block = false
258
+ next if @stopping || child.retiring
259
+
260
+ index = child.index
261
+ @crashes[index] = (@os.now - child.started_at < MIN_UPTIME) ? @crashes[index] + 1 : 0
262
+ crashes = @crashes.fetch(index)
263
+ delay = crashes.zero? ? 0 : [2**crashes, MAX_DELAY].min
264
+ logger.warn("swarm child #{index} (pid #{pid}) exited #{status.exitstatus || status.termsig}; " \
265
+ "restarting in #{delay}s")
266
+ Events.emit(@config, "swarm.child_exited", index: index, pid: pid, status: status.exitstatus || status.termsig, delay: delay)
267
+ @restarts[index] = @os.now + delay
268
+ end
269
+ end
270
+
271
+ def start_due_restarts
272
+ now = @os.now
273
+ @restarts.select { |_, at| at <= now }.each_key do |index|
274
+ @restarts.delete(index)
275
+ start_child(index)
276
+ end
277
+ end
278
+
279
+ private
280
+
281
+ def logger = @config.logger
282
+
283
+ # Is the child heartbeating in Redis yet?
284
+ def ready?(child)
285
+ wait_for(READY_TIMEOUT, poll: 1) do
286
+ return false unless @children.key?(child.pid) # it died
287
+
288
+ ProcessSet.new(config: @config).any? { |p| p["pid"] == child.pid && p["hostname"] == Robin.hostname }
289
+ end
290
+ end
291
+
292
+ def stop_child(child)
293
+ @os.kill(child.pid, "TERM")
294
+ return if wait_for { !@children.key?(child.pid) }
295
+
296
+ @os.kill(child.pid, "KILL")
297
+ reap(block: true) while @children.key?(child.pid)
298
+ end
299
+
300
+ # Polls, reaping exited children, until the block is true or +seconds+
301
+ # pass. Returns whether the block became true.
302
+ def wait_for(seconds = @config.timeout + 10, poll: 0.1)
303
+ deadline = @os.now + seconds
304
+ loop do
305
+ reap
306
+ return true if yield
307
+ return false if @os.now > deadline
308
+
309
+ @os.sleep(poll)
310
+ end
311
+ end
312
+
313
+ def signal_children(signal) = @children.each_key { |pid| @os.kill(pid, signal) }
314
+ end
315
+ end
316
+
317
+ # Spawned swarm children load this file with `ruby -r robin/swarm` so they
318
+ # stop when the swarm does.
319
+ Robin::Swarm.watch_parent_from_env
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+
5
+ module Robin
6
+ # Tells systemd how the server is doing, when it runs as a Type=notify
7
+ # service (systemd sets NOTIFY_SOCKET):
8
+ #
9
+ # [Service]
10
+ # Type=notify
11
+ # WatchdogSec=30
12
+ # ExecStart=/usr/bin/bundle exec robin
13
+ #
14
+ # The server reports READY=1 once it's working, STOPPING=1 when it
15
+ # starts shutting down, and, with WatchdogSec, pets the watchdog from its
16
+ # heartbeat (so keep WatchdogSec at several heartbeat intervals). Without
17
+ # NOTIFY_SOCKET every call is a no-op.
18
+ module Systemd
19
+ # Sends +state+ (e.g. "READY=1"). True if it was sent.
20
+ def self.notify(state, env: ENV)
21
+ path = env["NOTIFY_SOCKET"].to_s
22
+ return false if path.empty?
23
+
24
+ Socket.open(:UNIX, :DGRAM) do |sock|
25
+ sock.connect(Socket.sockaddr_un(address(path)))
26
+ sock.write(state)
27
+ end
28
+ true
29
+ rescue SystemCallError, IOError, ArgumentError, SocketError # (JRuby has no datagram Unix sockets)
30
+ false
31
+ end
32
+
33
+ # A leading "@" means Linux's abstract socket namespace.
34
+ def self.address(path)
35
+ path.start_with?("@") ? "\0#{path[1..]}" : path
36
+ end
37
+
38
+ # Whether systemd watches this process (WatchdogSec is set, and is
39
+ # meant for this pid).
40
+ def self.watchdog?(env: ENV, pid: Process.pid)
41
+ return false unless env["WATCHDOG_USEC"].to_i.positive?
42
+
43
+ watched = env["WATCHDOG_PID"].to_s
44
+ watched.empty? || Integer(watched, exception: false) == pid
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,158 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "monitor"
4
+ require "robin"
5
+
6
+ module Robin
7
+ # Test helpers. Requiring this file switches Robin to fake mode: pushes
8
+ # are kept in memory instead of going to Redis.
9
+ #
10
+ # require "robin/testing"
11
+ #
12
+ # HardJob.perform_async(1)
13
+ # assert_equal 1, HardJob.jobs.size
14
+ # HardJob.drain # runs them (and anything they push)
15
+ #
16
+ # Robin::Testing.inline! do # run jobs immediately when pushed
17
+ # HardJob.perform_async(2)
18
+ # end
19
+ #
20
+ # Robin::Testing.disable! # real Redis again
21
+ #
22
+ # Called with a block, a mode lasts for the block and only applies to
23
+ # the current thread (so tests running in parallel don't disturb each
24
+ # other); without one, it's the default for every thread.
25
+ #
26
+ # Client middleware runs in every mode; server middleware runs when
27
+ # fake jobs are drained and in inline mode.
28
+ module Testing
29
+ class EmptyQueue < Robin::Error; end
30
+
31
+ @mode = :fake
32
+ @queues = Hash.new { |h, k| h[k] = [] }
33
+ @mutex = Monitor.new
34
+ MODE_KEY = :robin_testing_mode
35
+
36
+ class << self
37
+ # The current thread's mode.
38
+ def mode = Thread.current[MODE_KEY] || @mode
39
+
40
+ def fake!(&) = switch(:fake, &)
41
+
42
+ def inline!(&) = switch(:inline, &)
43
+
44
+ def disable!(&) = switch(:disable, &)
45
+
46
+ def fake? = mode == :fake
47
+
48
+ def inline? = mode == :inline
49
+
50
+ def disabled? = mode == :disable
51
+
52
+ # Every faked job, in push order, keyed by queue.
53
+ attr_reader :queues
54
+
55
+ # Every faked job across all queues.
56
+ def jobs = @mutex.synchronize { @queues.values.flatten }
57
+
58
+ def jobs_for(class_name)
59
+ @mutex.synchronize { @queues.values.flatten.select { |j| j.fetch("class") == class_name } }
60
+ end
61
+
62
+ def clear_all = @mutex.synchronize { @queues.clear }
63
+
64
+ def clear(class_name)
65
+ @mutex.synchronize { @queues.each_value { |list| list.reject! { |j| j.fetch("class") == class_name } } }
66
+ end
67
+
68
+ # Runs faked jobs (including any they push) until none are left.
69
+ def drain_all
70
+ while (job = shift_next)
71
+ execute(job)
72
+ end
73
+ end
74
+
75
+ # Runs faked jobs of one class until none are left.
76
+ def drain(class_name)
77
+ while (job = shift_next(class_name))
78
+ execute(job)
79
+ end
80
+ end
81
+
82
+ def perform_one(class_name)
83
+ job = shift_next(class_name) or raise EmptyQueue, "no #{class_name} jobs to perform"
84
+ execute(job)
85
+ end
86
+
87
+ # Called by the client in place of a Redis push.
88
+ def push(payloads)
89
+ payloads.each do |payload|
90
+ job = Robin.load_json(Robin.dump_json(payload))
91
+ job["enqueued_at"] ||= Robin.now_ms unless job["at"]
92
+ if inline?
93
+ execute(job)
94
+ else
95
+ @mutex.synchronize { @queues[job.fetch("queue")] << job }
96
+ end
97
+ end
98
+ end
99
+
100
+ # Runs one job hash through the server middleware chain.
101
+ def execute(payload, config: Robin.config)
102
+ klass = Object.const_get(payload.fetch("class"))
103
+ job = klass.new
104
+ job.jid = payload.fetch("jid") if job.respond_to?(:jid=)
105
+ job.payload = payload if job.respond_to?(:payload=)
106
+ config.server_middleware.invoke(job, payload, payload["queue"]) do
107
+ job.perform(*payload.fetch("args"))
108
+ end
109
+ end
110
+
111
+ private
112
+
113
+ def switch(mode)
114
+ unless block_given?
115
+ @mode = mode
116
+ return
117
+ end
118
+
119
+ previous = Thread.current[MODE_KEY]
120
+ Thread.current[MODE_KEY] = mode
121
+ begin
122
+ yield
123
+ ensure
124
+ Thread.current[MODE_KEY] = previous
125
+ end
126
+ end
127
+
128
+ def shift_next(class_name = nil)
129
+ @mutex.synchronize do
130
+ candidates = @queues.values.flatten
131
+ candidates = candidates.select { |j| j.fetch("class") == class_name } if class_name
132
+ job = candidates.min_by { |j| j["at"] ? Robin.epoch_ms(j["at"]) : j["enqueued_at"] || 0 }
133
+ @queues.fetch(job.fetch("queue")).delete(job) if job
134
+ end
135
+ end
136
+ end
137
+
138
+ # Routes pushes to memory unless testing is disabled. Returns the
139
+ # payloads it handled, or nil to let Redis take them (see
140
+ # Client.interceptor).
141
+ def self.intercept(payloads) = (push(payloads) unless disabled?)
142
+
143
+ # Class-level helpers on every Robin::Job class.
144
+ module JobMethods
145
+ def jobs = Testing.jobs_for(name)
146
+
147
+ def clear = Testing.clear(name)
148
+
149
+ def drain = Testing.drain(name)
150
+
151
+ def perform_one = Testing.perform_one(name)
152
+ end
153
+ end
154
+
155
+ # (lambda, not ->: Steep splats a -> lambda's one Array parameter.)
156
+ Client.interceptor = lambda { |payloads| Testing.intercept(payloads) }
157
+ Job::ClassMethods.include(Testing::JobMethods)
158
+ end
data/lib/robin/top.rb ADDED
@@ -0,0 +1,178 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "io/console"
4
+
5
+ module Robin
6
+ # A live view of a cluster in the terminal: `robin top`.
7
+ #
8
+ # Three views, refreshed every +interval+ seconds: the overview (totals
9
+ # and queues), processes, and the jobs running right now. Keys: 1, 2
10
+ # and 3 (or tab) switch views, r refreshes, q quits. It only reads, so
11
+ # it's safe to leave running anywhere.
12
+ class Top
13
+ VIEWS = %w[Overview Processes Busy].freeze
14
+ QUIT = ["q", "Q", "\u0003", "\u0004"].freeze
15
+ DEFAULT_SIZE = [24, 80] #: [Integer, Integer]
16
+ .freeze
17
+ # Bytes of key presses read at once.
18
+ KEY_BUFFER = 64
19
+
20
+ # Everything one frame shows, read from Redis in one go.
21
+ Snapshot = Struct.new(:stats, :queues, :processes, :work, :at)
22
+
23
+ class Snapshot
24
+ def self.take(config)
25
+ paused = Queue.paused(config: config)
26
+ queues = Queue.all(config: config).map { |q| [q.name, q.size, q.latency, paused.include?(q.name)] } #: Array[Top::queue_row]
27
+ new(Stats.new(config: config), queues.sort_by { |name, size| [-size, name] },
28
+ ProcessSet.new(config: config).to_a, WorkSet.new(config: config).to_a, Robin.now_ms)
29
+ end
30
+ end
31
+
32
+ def initialize(config: Robin.config, input: $stdin, output: $stdout, interval: 2)
33
+ @config = config
34
+ @input = input
35
+ @output = output
36
+ @interval = interval
37
+ @view = 0
38
+ end
39
+
40
+ attr_reader :view
41
+
42
+ # Runs until a quit key (or the end of input).
43
+ def run
44
+ @output.write("\e[?1049h\e[?25l")
45
+ @input.tty? ? @input.raw { loop_frames } : loop_frames
46
+ ensure
47
+ @output.write("\e[?25h\e[?1049l")
48
+ @output.flush
49
+ end
50
+
51
+ # One frame for a terminal +height+ lines by +width+ columns.
52
+ def frame(height, width)
53
+ lines = begin
54
+ body(Snapshot.take(@config))
55
+ rescue => e
56
+ ["", "Can't read from Redis: #{e.class}: #{e}", "Retrying every #{@interval}s."]
57
+ end
58
+ [header(width), tabs, *lines].first(height).map { |line| fit(line, width) }
59
+ end
60
+
61
+ # Acts on a key press; false means quit.
62
+ def key(char)
63
+ case char
64
+ when *QUIT then return false
65
+ when "1", "2", "3" then @view = Integer(char) - 1
66
+ when "\t" then @view = (@view + 1) % VIEWS.size
67
+ end
68
+ true
69
+ end
70
+
71
+ private
72
+
73
+ def loop_frames
74
+ loop do
75
+ height, width = size
76
+ @output.write("\e[H", frame(height, width).map { |l| "#{l}\e[K" }.join("\r\n"), "\e[J")
77
+ @output.flush
78
+ next unless @input.wait_readable(@interval)
79
+
80
+ chars = @input.read_nonblock(KEY_BUFFER, exception: false)
81
+ break if chars.nil? # the end of input
82
+ next if chars == :wait_readable
83
+ break unless chars.each_char.all? { |c| key(c) }
84
+ end
85
+ end
86
+
87
+ def size
88
+ height, width = @output.winsize
89
+ (height.positive? && width.positive?) ? [height, width] : DEFAULT_SIZE
90
+ rescue SystemCallError, NoMethodError
91
+ DEFAULT_SIZE
92
+ end
93
+
94
+ def header(width)
95
+ left = "Robin #{VERSION}"
96
+ right = Time.at(Rational(Robin.now_ms, 1000)).strftime("%H:%M:%S")
97
+ left + " " * [width - left.size - right.size, 1].max + right
98
+ end
99
+
100
+ def tabs
101
+ views = VIEWS.each_with_index.map { |name, i| (i == @view) ? "[#{i + 1} #{name}]" : " #{i + 1} #{name} " }
102
+ "#{views.join(" ")} tab next · r refresh · q quit"
103
+ end
104
+
105
+ def body(snap)
106
+ case @view
107
+ when 0 then overview(snap)
108
+ when 1 then processes(snap)
109
+ else busy(snap)
110
+ end
111
+ end
112
+
113
+ def overview(snap)
114
+ s = snap.stats
115
+ busy = "#{s.workers_size}/#{snap.processes.sum { |p| p["concurrency"].to_i }}"
116
+ [
117
+ "",
118
+ pairs({"Processed" => number(s.processed), "Failed" => number(s.failed), "Busy" => busy,
119
+ "Enqueued" => number(s.enqueued)}),
120
+ pairs({"Scheduled" => number(s.scheduled_size), "Retries" => number(s.retry_size),
121
+ "Dead" => number(s.dead_size), "Processes" => number(snap.processes.size)}),
122
+ ""
123
+ ] + table(%w[QUEUE SIZE LATENCY], snap.queues.map do |name, size, latency, paused|
124
+ [paused ? "#{name} (paused)" : name, number(size), duration(latency)]
125
+ end, right: [1, 2], empty: "No queues")
126
+ end
127
+
128
+ def processes(snap)
129
+ rows = snap.processes.map do |p|
130
+ flags = [("leader" if p.leader?), ("quiet" if p.quiet?)].compact.join(", ")
131
+ name = [p["hostname"], p["pid"]].compact.join(":")
132
+ name = "#{name} [#{p["tag"]}]" if p["tag"]
133
+ [name, "#{p.busy}/#{p["concurrency"].to_i}", megabytes(p.rss), since(snap.at, p.started_at),
134
+ since(snap.at, p.beat), flags, p["queues"].to_a.join(", ")]
135
+ end
136
+ [""] + table(%w[PROCESS BUSY RSS UP BEAT STATUS QUEUES], rows, right: [1, 2, 3, 4], empty: "No processes")
137
+ end
138
+
139
+ def busy(snap)
140
+ rows = snap.work.sort_by { |_, _, work| work.fetch("run_at") }.map do |identity, tid, work|
141
+ job = JobRecord.new(work.fetch("payload"))
142
+ [identity, tid, work.fetch("queue"), job.display_class, since(snap.at, work.fetch("run_at")), Robin.dump_json(job.display_args)]
143
+ end
144
+ [""] + table(%w[PROCESS THREAD QUEUE JOB RUNNING ARGS], rows, right: [4], empty: "Nothing running")
145
+ end
146
+
147
+ # Columns padded to their widest cell; +right+ lists the columns
148
+ # aligned right (numbers).
149
+ def table(headers, rows, right:, empty:)
150
+ return [" #{empty}"] if rows.empty?
151
+
152
+ widths = headers.each_index.map { |i| [headers, *rows].map { |r| r.fetch(i).size }.max }
153
+ [headers, *rows].map do |row|
154
+ row.each_with_index.map { |cell, i| right.include?(i) ? cell.rjust(widths.fetch(i)) : cell.ljust(widths.fetch(i)) }.join(" ").rstrip
155
+ end
156
+ end
157
+
158
+ def pairs(hash) = hash.map { |label, value| "#{label} #{value}" }.join(" ")
159
+
160
+ def fit(line, width) = (line.size > width) ? "#{line[0, width - 1]}…" : line
161
+
162
+ def number(n) = n.to_i.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
163
+
164
+ def megabytes(kb) = "#{(kb / 1024.0).round} MB"
165
+
166
+ # How long ago +ms+ (epoch milliseconds) was, before +now+.
167
+ def since(now, ms) = ms ? duration((now - ms) / 1000.0) : "-"
168
+
169
+ def duration(seconds)
170
+ seconds = [seconds.to_f, 0.0].max
171
+ if seconds < 60 then "#{seconds.round(1)}s"
172
+ elsif seconds < 3600 then "#{(seconds / 60).floor}m #{(seconds % 60).floor}s"
173
+ elsif seconds < 86_400 then "#{(seconds / 3600).floor}h #{(seconds % 3600 / 60).floor}m"
174
+ else "#{(seconds / 86_400).floor}d #{(seconds % 86_400 / 3600).floor}h"
175
+ end
176
+ end
177
+ end
178
+ end