robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
data/lib/robin/swarm.rb
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "etc"
|
|
4
|
+
require "rbconfig"
|
|
5
|
+
require "socket"
|
|
6
|
+
|
|
7
|
+
module Robin
|
|
8
|
+
# Runs several Robin processes on one machine from a single command.
|
|
9
|
+
#
|
|
10
|
+
# robin swarm -n 4 -r ./config/environment.rb -c 10
|
|
11
|
+
#
|
|
12
|
+
# Where fork is available, the parent boots the application once and
|
|
13
|
+
# forks children, so they share memory copy-on-write. Elsewhere (JRuby)
|
|
14
|
+
# each child is a fresh `robin` process started with the same options.
|
|
15
|
+
# Dead children are replaced, with a growing delay if they keep crashing.
|
|
16
|
+
# Signals sent to the parent:
|
|
17
|
+
#
|
|
18
|
+
# TERM, INT:: graceful shutdown of every child, then exit
|
|
19
|
+
# TSTP:: quiet every child (finish running jobs, fetch no more)
|
|
20
|
+
# TTIN:: every child logs its thread backtraces
|
|
21
|
+
# USR2:: rolling restart: each child in turn is replaced by a fresh
|
|
22
|
+
# one, which must be up (heartbeating) *before* the old one
|
|
23
|
+
# stops, so capacity never drops
|
|
24
|
+
#
|
|
25
|
+
# The number of children comes from -n, else ROBIN_SWARM_COUNT, else the
|
|
26
|
+
# number of CPU cores. A decimal means that many per core: 0.5 on 8 cores
|
|
27
|
+
# is 4 children, 1.5 is 12 (always at least 1).
|
|
28
|
+
#
|
|
29
|
+
# Children see ROBIN_SWARM_INDEX (0...n) and stop by themselves if the
|
|
30
|
+
# parent dies. A child exceeding config.memory_limit_mb stops gracefully
|
|
31
|
+
# and is replaced.
|
|
32
|
+
class Swarm
|
|
33
|
+
READY_TIMEOUT = 60
|
|
34
|
+
MIN_UPTIME = 10
|
|
35
|
+
MAX_DELAY = 60
|
|
36
|
+
SIGNALS = %w[INT TERM TSTP TTIN USR2 CHLD].freeze
|
|
37
|
+
EXE = File.expand_path("../../exe/robin", __dir__)
|
|
38
|
+
LIB = File.expand_path("..", __dir__)
|
|
39
|
+
|
|
40
|
+
Child = Struct.new(:pid, :index, :started_at, :retiring)
|
|
41
|
+
|
|
42
|
+
# Operating-system primitives, separated so the supervisor logic can
|
|
43
|
+
# be tested without real processes.
|
|
44
|
+
class OS
|
|
45
|
+
SIGNAL_BUFFER = 1024
|
|
46
|
+
|
|
47
|
+
# mutant:disable -- only differs where fork is missing (JRuby);
|
|
48
|
+
# mutant runs on MRI.
|
|
49
|
+
def fork? = Process.respond_to?(:fork)
|
|
50
|
+
|
|
51
|
+
# mutant:disable -- `super` reaches Kernel#fork, the same call.
|
|
52
|
+
def fork(&) = Process.fork(&)
|
|
53
|
+
|
|
54
|
+
# mutant:disable -- `super` reaches Kernel#spawn, the same call.
|
|
55
|
+
def spawn(env, *command) = Process.spawn(env, *command)
|
|
56
|
+
|
|
57
|
+
# Tidies the heap (GC, compaction) so forked children share more
|
|
58
|
+
# memory copy-on-write.
|
|
59
|
+
def warmup = Process.warmup
|
|
60
|
+
|
|
61
|
+
def kill(pid, signal)
|
|
62
|
+
Process.kill(signal, pid)
|
|
63
|
+
rescue Errno::ESRCH
|
|
64
|
+
nil
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# [pid, status] of an exited child, or nil.
|
|
68
|
+
def wait(block: false)
|
|
69
|
+
wait2(block)
|
|
70
|
+
rescue Errno::ECHILD
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
75
|
+
|
|
76
|
+
# mutant:disable -- `super` reaches Kernel#sleep, the same call.
|
|
77
|
+
def sleep(seconds) = Kernel.sleep(seconds)
|
|
78
|
+
|
|
79
|
+
# Routes +signals+ into a pipe; returns its reading end.
|
|
80
|
+
def trap_signals(signals)
|
|
81
|
+
reader, writer = IO.pipe
|
|
82
|
+
signals.each { |sig| trap(sig) { notify(writer, sig) } }
|
|
83
|
+
reader
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def reset_signals(signals) = signals.each { |sig| trap(sig, "DEFAULT") }
|
|
87
|
+
|
|
88
|
+
# Signals received within +timeout+ seconds.
|
|
89
|
+
def read_signals(reader, timeout)
|
|
90
|
+
return [] unless reader.wait_readable(timeout)
|
|
91
|
+
|
|
92
|
+
reader.read_nonblock(SIGNAL_BUFFER).lines(chomp: true)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# mutant:disable -- `self.exit!` reaches Kernel#exit!, the same call.
|
|
96
|
+
def finish_child = Kernel.exit!(0)
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
# mutant:disable -- equivalent spellings: wait2's pid -1 or omitted,
|
|
101
|
+
# flags 0 or nil.
|
|
102
|
+
def wait2(block) = Process.wait2(-1, block ? 0 : Process::WNOHANG)
|
|
103
|
+
|
|
104
|
+
# mutant:disable -- raising instead only differs once 64KB of
|
|
105
|
+
# unread signal names fill the pipe.
|
|
106
|
+
def notify(writer, signal) = writer.write_nonblock("#{signal}\n", exception: false)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Stops this process when +parent+ goes away. Children call this; with
|
|
110
|
+
# spawn they get here by loading this file (see #child_command).
|
|
111
|
+
def self.watch_parent(parent, os: OS.new, interval: 1)
|
|
112
|
+
Thread.new do
|
|
113
|
+
Thread.current.name = "robin.swarm.watch"
|
|
114
|
+
os.sleep(interval) while Process.ppid == parent
|
|
115
|
+
os.kill(Process.pid, "TERM")
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Watches the swarm parent named in ROBIN_SWARM_PARENT, if it is ours.
|
|
120
|
+
# Children for a count setting (see the class comment).
|
|
121
|
+
def self.process_count(value, cores = Etc.nprocessors)
|
|
122
|
+
return cores if value.nil?
|
|
123
|
+
|
|
124
|
+
text = value.to_s
|
|
125
|
+
number = text.include?(".") ? Float(text) : Integer(text)
|
|
126
|
+
raise ArgumentError, "the swarm's process count must be positive: #{text}" unless number.positive?
|
|
127
|
+
|
|
128
|
+
number.integer? ? number : [(number * cores).round, 1].max #: Integer
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def self.watch_parent_from_env(env = ENV, os: OS.new)
|
|
132
|
+
parent = env["ROBIN_SWARM_PARENT"]
|
|
133
|
+
watch_parent(Process.ppid, os: os) if parent && Integer(parent) == Process.ppid
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
attr_reader :children, :count, :mode
|
|
137
|
+
|
|
138
|
+
def initialize(cli, os: OS.new)
|
|
139
|
+
@cli = cli
|
|
140
|
+
@config = cli.config
|
|
141
|
+
@os = os
|
|
142
|
+
@count = Swarm.process_count(cli.options[:processes] || ENV["ROBIN_SWARM_COUNT"])
|
|
143
|
+
@mode = ENV["ROBIN_SWARM_MODE"] || (os.fork? ? "fork" : "spawn")
|
|
144
|
+
@children = {}
|
|
145
|
+
@crashes = Hash.new(0)
|
|
146
|
+
@restarts = {} # index => when (monotonic) to start a replacement
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def run
|
|
150
|
+
@cli.boot
|
|
151
|
+
if @mode == "fork"
|
|
152
|
+
# Loaded before forking, the code is shared copy-on-write instead
|
|
153
|
+
# of autoloaded by each child.
|
|
154
|
+
Robin.eager_load!
|
|
155
|
+
@os.warmup
|
|
156
|
+
end
|
|
157
|
+
@signals = @os.trap_signals(SIGNALS)
|
|
158
|
+
logger.info("swarm #{Process.pid} starting #{@count} children (#{@mode})")
|
|
159
|
+
@count.times { |i| start_child(i) }
|
|
160
|
+
until @stopping # shutdown returns once every child is gone
|
|
161
|
+
reap
|
|
162
|
+
start_due_restarts
|
|
163
|
+
@os.read_signals(@signals, 1).each { |sig| handle(sig) }
|
|
164
|
+
end
|
|
165
|
+
logger.info("swarm #{Process.pid} stopped")
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def handle(signal)
|
|
169
|
+
case signal
|
|
170
|
+
when "INT", "TERM" then shutdown
|
|
171
|
+
when "TSTP", "TTIN" then signal_children(signal)
|
|
172
|
+
when "USR2" then rolling_restart
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def shutdown
|
|
177
|
+
return if @stopping
|
|
178
|
+
|
|
179
|
+
@stopping = true
|
|
180
|
+
@restarts.clear
|
|
181
|
+
logger.info("swarm stopping #{@children.size} children")
|
|
182
|
+
signal_children("TERM")
|
|
183
|
+
return if wait_for { @children.empty? }
|
|
184
|
+
|
|
185
|
+
logger.warn("killing #{@children.size} children that didn't stop in time")
|
|
186
|
+
signal_children("KILL")
|
|
187
|
+
reap(block: true) until @children.empty?
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def rolling_restart
|
|
191
|
+
return if @stopping
|
|
192
|
+
|
|
193
|
+
logger.info("swarm rolling restart")
|
|
194
|
+
@children.values.sort_by(&:index).each do |old|
|
|
195
|
+
next unless @children.key?(old.pid)
|
|
196
|
+
|
|
197
|
+
old.retiring = true
|
|
198
|
+
replacement = start_child(old.index)
|
|
199
|
+
if ready?(replacement)
|
|
200
|
+
stop_child(old)
|
|
201
|
+
else
|
|
202
|
+
logger.error("replacement for child #{old.index} didn't start; keeping the old one")
|
|
203
|
+
replacement.retiring = true
|
|
204
|
+
@os.kill(replacement.pid, "KILL")
|
|
205
|
+
old.retiring = false
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# The options a spawned child needs to run like this process.
|
|
211
|
+
def child_arguments
|
|
212
|
+
o = @cli.options
|
|
213
|
+
args = [] #: Array[String]
|
|
214
|
+
{require: "-r", config_file: "-C", environment: "-e", concurrency: "-c", timeout: "-t", tag: "-g"}.each do |key, flag|
|
|
215
|
+
args.push(flag, o.fetch(key).to_s) if o[key]
|
|
216
|
+
end
|
|
217
|
+
Array(o[:queues]).each { |q| args.push("-q", [q].join(",")) }
|
|
218
|
+
args << "-v" if o[:verbose]
|
|
219
|
+
args << "--json-logs" if o[:json_logs]
|
|
220
|
+
args
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def child_command = [RbConfig.ruby, "-r", "robin/swarm", EXE, *child_arguments]
|
|
224
|
+
|
|
225
|
+
def child_env(index)
|
|
226
|
+
{"ROBIN_SWARM_INDEX" => index.to_s, "ROBIN_SWARM_PARENT" => Process.pid.to_s,
|
|
227
|
+
"RUBYLIB" => [LIB, ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR)}
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def start_child(index)
|
|
231
|
+
pid = (@mode == "fork") ? @os.fork { child_main(index) } : @os.spawn(child_env(index), *child_command)
|
|
232
|
+
child = Child.new(pid, index, @os.now, false)
|
|
233
|
+
@children[pid] = child
|
|
234
|
+
logger.info("swarm child #{index} started (pid #{pid})")
|
|
235
|
+
child
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# Runs inside a forked child.
|
|
239
|
+
def child_main(index)
|
|
240
|
+
@os.reset_signals(SIGNALS)
|
|
241
|
+
ENV["ROBIN_SWARM_INDEX"] = index.to_s
|
|
242
|
+
Swarm.watch_parent(Process.ppid, os: @os)
|
|
243
|
+
@cli.config.fire_event(:fork)
|
|
244
|
+
@cli.serve
|
|
245
|
+
@os.finish_child
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
# Collects exited children and schedules replacements.
|
|
249
|
+
def reap(block: false)
|
|
250
|
+
loop do
|
|
251
|
+
waited = @os.wait(block: block) or break
|
|
252
|
+
pid, status = waited
|
|
253
|
+
|
|
254
|
+
child = @children.delete(pid)
|
|
255
|
+
next unless child
|
|
256
|
+
|
|
257
|
+
block = false
|
|
258
|
+
next if @stopping || child.retiring
|
|
259
|
+
|
|
260
|
+
index = child.index
|
|
261
|
+
@crashes[index] = (@os.now - child.started_at < MIN_UPTIME) ? @crashes[index] + 1 : 0
|
|
262
|
+
crashes = @crashes.fetch(index)
|
|
263
|
+
delay = crashes.zero? ? 0 : [2**crashes, MAX_DELAY].min
|
|
264
|
+
logger.warn("swarm child #{index} (pid #{pid}) exited #{status.exitstatus || status.termsig}; " \
|
|
265
|
+
"restarting in #{delay}s")
|
|
266
|
+
Events.emit(@config, "swarm.child_exited", index: index, pid: pid, status: status.exitstatus || status.termsig, delay: delay)
|
|
267
|
+
@restarts[index] = @os.now + delay
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def start_due_restarts
|
|
272
|
+
now = @os.now
|
|
273
|
+
@restarts.select { |_, at| at <= now }.each_key do |index|
|
|
274
|
+
@restarts.delete(index)
|
|
275
|
+
start_child(index)
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
private
|
|
280
|
+
|
|
281
|
+
def logger = @config.logger
|
|
282
|
+
|
|
283
|
+
# Is the child heartbeating in Redis yet?
|
|
284
|
+
def ready?(child)
|
|
285
|
+
wait_for(READY_TIMEOUT, poll: 1) do
|
|
286
|
+
return false unless @children.key?(child.pid) # it died
|
|
287
|
+
|
|
288
|
+
ProcessSet.new(config: @config).any? { |p| p["pid"] == child.pid && p["hostname"] == Robin.hostname }
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def stop_child(child)
|
|
293
|
+
@os.kill(child.pid, "TERM")
|
|
294
|
+
return if wait_for { !@children.key?(child.pid) }
|
|
295
|
+
|
|
296
|
+
@os.kill(child.pid, "KILL")
|
|
297
|
+
reap(block: true) while @children.key?(child.pid)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# Polls, reaping exited children, until the block is true or +seconds+
|
|
301
|
+
# pass. Returns whether the block became true.
|
|
302
|
+
def wait_for(seconds = @config.timeout + 10, poll: 0.1)
|
|
303
|
+
deadline = @os.now + seconds
|
|
304
|
+
loop do
|
|
305
|
+
reap
|
|
306
|
+
return true if yield
|
|
307
|
+
return false if @os.now > deadline
|
|
308
|
+
|
|
309
|
+
@os.sleep(poll)
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def signal_children(signal) = @children.each_key { |pid| @os.kill(pid, signal) }
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Spawned swarm children load this file with `ruby -r robin/swarm` so they
|
|
318
|
+
# stop when the swarm does.
|
|
319
|
+
Robin::Swarm.watch_parent_from_env
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Tells systemd how the server is doing, when it runs as a Type=notify
|
|
7
|
+
# service (systemd sets NOTIFY_SOCKET):
|
|
8
|
+
#
|
|
9
|
+
# [Service]
|
|
10
|
+
# Type=notify
|
|
11
|
+
# WatchdogSec=30
|
|
12
|
+
# ExecStart=/usr/bin/bundle exec robin
|
|
13
|
+
#
|
|
14
|
+
# The server reports READY=1 once it's working, STOPPING=1 when it
|
|
15
|
+
# starts shutting down, and, with WatchdogSec, pets the watchdog from its
|
|
16
|
+
# heartbeat (so keep WatchdogSec at several heartbeat intervals). Without
|
|
17
|
+
# NOTIFY_SOCKET every call is a no-op.
|
|
18
|
+
module Systemd
|
|
19
|
+
# Sends +state+ (e.g. "READY=1"). True if it was sent.
|
|
20
|
+
def self.notify(state, env: ENV)
|
|
21
|
+
path = env["NOTIFY_SOCKET"].to_s
|
|
22
|
+
return false if path.empty?
|
|
23
|
+
|
|
24
|
+
Socket.open(:UNIX, :DGRAM) do |sock|
|
|
25
|
+
sock.connect(Socket.sockaddr_un(address(path)))
|
|
26
|
+
sock.write(state)
|
|
27
|
+
end
|
|
28
|
+
true
|
|
29
|
+
rescue SystemCallError, IOError, ArgumentError, SocketError # (JRuby has no datagram Unix sockets)
|
|
30
|
+
false
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# A leading "@" means Linux's abstract socket namespace.
|
|
34
|
+
def self.address(path)
|
|
35
|
+
path.start_with?("@") ? "\0#{path[1..]}" : path
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Whether systemd watches this process (WatchdogSec is set, and is
|
|
39
|
+
# meant for this pid).
|
|
40
|
+
def self.watchdog?(env: ENV, pid: Process.pid)
|
|
41
|
+
return false unless env["WATCHDOG_USEC"].to_i.positive?
|
|
42
|
+
|
|
43
|
+
watched = env["WATCHDOG_PID"].to_s
|
|
44
|
+
watched.empty? || Integer(watched, exception: false) == pid
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "monitor"
|
|
4
|
+
require "robin"
|
|
5
|
+
|
|
6
|
+
module Robin
|
|
7
|
+
# Test helpers. Requiring this file switches Robin to fake mode: pushes
|
|
8
|
+
# are kept in memory instead of going to Redis.
|
|
9
|
+
#
|
|
10
|
+
# require "robin/testing"
|
|
11
|
+
#
|
|
12
|
+
# HardJob.perform_async(1)
|
|
13
|
+
# assert_equal 1, HardJob.jobs.size
|
|
14
|
+
# HardJob.drain # runs them (and anything they push)
|
|
15
|
+
#
|
|
16
|
+
# Robin::Testing.inline! do # run jobs immediately when pushed
|
|
17
|
+
# HardJob.perform_async(2)
|
|
18
|
+
# end
|
|
19
|
+
#
|
|
20
|
+
# Robin::Testing.disable! # real Redis again
|
|
21
|
+
#
|
|
22
|
+
# Called with a block, a mode lasts for the block and only applies to
|
|
23
|
+
# the current thread (so tests running in parallel don't disturb each
|
|
24
|
+
# other); without one, it's the default for every thread.
|
|
25
|
+
#
|
|
26
|
+
# Client middleware runs in every mode; server middleware runs when
|
|
27
|
+
# fake jobs are drained and in inline mode.
|
|
28
|
+
module Testing
|
|
29
|
+
class EmptyQueue < Robin::Error; end
|
|
30
|
+
|
|
31
|
+
@mode = :fake
|
|
32
|
+
@queues = Hash.new { |h, k| h[k] = [] }
|
|
33
|
+
@mutex = Monitor.new
|
|
34
|
+
MODE_KEY = :robin_testing_mode
|
|
35
|
+
|
|
36
|
+
class << self
|
|
37
|
+
# The current thread's mode.
|
|
38
|
+
def mode = Thread.current[MODE_KEY] || @mode
|
|
39
|
+
|
|
40
|
+
def fake!(&) = switch(:fake, &)
|
|
41
|
+
|
|
42
|
+
def inline!(&) = switch(:inline, &)
|
|
43
|
+
|
|
44
|
+
def disable!(&) = switch(:disable, &)
|
|
45
|
+
|
|
46
|
+
def fake? = mode == :fake
|
|
47
|
+
|
|
48
|
+
def inline? = mode == :inline
|
|
49
|
+
|
|
50
|
+
def disabled? = mode == :disable
|
|
51
|
+
|
|
52
|
+
# Every faked job, in push order, keyed by queue.
|
|
53
|
+
attr_reader :queues
|
|
54
|
+
|
|
55
|
+
# Every faked job across all queues.
|
|
56
|
+
def jobs = @mutex.synchronize { @queues.values.flatten }
|
|
57
|
+
|
|
58
|
+
def jobs_for(class_name)
|
|
59
|
+
@mutex.synchronize { @queues.values.flatten.select { |j| j.fetch("class") == class_name } }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def clear_all = @mutex.synchronize { @queues.clear }
|
|
63
|
+
|
|
64
|
+
def clear(class_name)
|
|
65
|
+
@mutex.synchronize { @queues.each_value { |list| list.reject! { |j| j.fetch("class") == class_name } } }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Runs faked jobs (including any they push) until none are left.
|
|
69
|
+
def drain_all
|
|
70
|
+
while (job = shift_next)
|
|
71
|
+
execute(job)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Runs faked jobs of one class until none are left.
|
|
76
|
+
def drain(class_name)
|
|
77
|
+
while (job = shift_next(class_name))
|
|
78
|
+
execute(job)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def perform_one(class_name)
|
|
83
|
+
job = shift_next(class_name) or raise EmptyQueue, "no #{class_name} jobs to perform"
|
|
84
|
+
execute(job)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Called by the client in place of a Redis push.
|
|
88
|
+
def push(payloads)
|
|
89
|
+
payloads.each do |payload|
|
|
90
|
+
job = Robin.load_json(Robin.dump_json(payload))
|
|
91
|
+
job["enqueued_at"] ||= Robin.now_ms unless job["at"]
|
|
92
|
+
if inline?
|
|
93
|
+
execute(job)
|
|
94
|
+
else
|
|
95
|
+
@mutex.synchronize { @queues[job.fetch("queue")] << job }
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# Runs one job hash through the server middleware chain.
|
|
101
|
+
def execute(payload, config: Robin.config)
|
|
102
|
+
klass = Object.const_get(payload.fetch("class"))
|
|
103
|
+
job = klass.new
|
|
104
|
+
job.jid = payload.fetch("jid") if job.respond_to?(:jid=)
|
|
105
|
+
job.payload = payload if job.respond_to?(:payload=)
|
|
106
|
+
config.server_middleware.invoke(job, payload, payload["queue"]) do
|
|
107
|
+
job.perform(*payload.fetch("args"))
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
private
|
|
112
|
+
|
|
113
|
+
def switch(mode)
|
|
114
|
+
unless block_given?
|
|
115
|
+
@mode = mode
|
|
116
|
+
return
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
previous = Thread.current[MODE_KEY]
|
|
120
|
+
Thread.current[MODE_KEY] = mode
|
|
121
|
+
begin
|
|
122
|
+
yield
|
|
123
|
+
ensure
|
|
124
|
+
Thread.current[MODE_KEY] = previous
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def shift_next(class_name = nil)
|
|
129
|
+
@mutex.synchronize do
|
|
130
|
+
candidates = @queues.values.flatten
|
|
131
|
+
candidates = candidates.select { |j| j.fetch("class") == class_name } if class_name
|
|
132
|
+
job = candidates.min_by { |j| j["at"] ? Robin.epoch_ms(j["at"]) : j["enqueued_at"] || 0 }
|
|
133
|
+
@queues.fetch(job.fetch("queue")).delete(job) if job
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Routes pushes to memory unless testing is disabled. Returns the
|
|
139
|
+
# payloads it handled, or nil to let Redis take them (see
|
|
140
|
+
# Client.interceptor).
|
|
141
|
+
def self.intercept(payloads) = (push(payloads) unless disabled?)
|
|
142
|
+
|
|
143
|
+
# Class-level helpers on every Robin::Job class.
|
|
144
|
+
module JobMethods
|
|
145
|
+
def jobs = Testing.jobs_for(name)
|
|
146
|
+
|
|
147
|
+
def clear = Testing.clear(name)
|
|
148
|
+
|
|
149
|
+
def drain = Testing.drain(name)
|
|
150
|
+
|
|
151
|
+
def perform_one = Testing.perform_one(name)
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# (lambda, not ->: Steep splats a -> lambda's one Array parameter.)
|
|
156
|
+
Client.interceptor = lambda { |payloads| Testing.intercept(payloads) }
|
|
157
|
+
Job::ClassMethods.include(Testing::JobMethods)
|
|
158
|
+
end
|
data/lib/robin/top.rb
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "io/console"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# A live view of a cluster in the terminal: `robin top`.
|
|
7
|
+
#
|
|
8
|
+
# Three views, refreshed every +interval+ seconds: the overview (totals
|
|
9
|
+
# and queues), processes, and the jobs running right now. Keys: 1, 2
|
|
10
|
+
# and 3 (or tab) switch views, r refreshes, q quits. It only reads, so
|
|
11
|
+
# it's safe to leave running anywhere.
|
|
12
|
+
class Top
|
|
13
|
+
VIEWS = %w[Overview Processes Busy].freeze
|
|
14
|
+
QUIT = ["q", "Q", "\u0003", "\u0004"].freeze
|
|
15
|
+
DEFAULT_SIZE = [24, 80] #: [Integer, Integer]
|
|
16
|
+
.freeze
|
|
17
|
+
# Bytes of key presses read at once.
|
|
18
|
+
KEY_BUFFER = 64
|
|
19
|
+
|
|
20
|
+
# Everything one frame shows, read from Redis in one go.
|
|
21
|
+
Snapshot = Struct.new(:stats, :queues, :processes, :work, :at)
|
|
22
|
+
|
|
23
|
+
class Snapshot
|
|
24
|
+
def self.take(config)
|
|
25
|
+
paused = Queue.paused(config: config)
|
|
26
|
+
queues = Queue.all(config: config).map { |q| [q.name, q.size, q.latency, paused.include?(q.name)] } #: Array[Top::queue_row]
|
|
27
|
+
new(Stats.new(config: config), queues.sort_by { |name, size| [-size, name] },
|
|
28
|
+
ProcessSet.new(config: config).to_a, WorkSet.new(config: config).to_a, Robin.now_ms)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def initialize(config: Robin.config, input: $stdin, output: $stdout, interval: 2)
|
|
33
|
+
@config = config
|
|
34
|
+
@input = input
|
|
35
|
+
@output = output
|
|
36
|
+
@interval = interval
|
|
37
|
+
@view = 0
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
attr_reader :view
|
|
41
|
+
|
|
42
|
+
# Runs until a quit key (or the end of input).
|
|
43
|
+
def run
|
|
44
|
+
@output.write("\e[?1049h\e[?25l")
|
|
45
|
+
@input.tty? ? @input.raw { loop_frames } : loop_frames
|
|
46
|
+
ensure
|
|
47
|
+
@output.write("\e[?25h\e[?1049l")
|
|
48
|
+
@output.flush
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# One frame for a terminal +height+ lines by +width+ columns.
|
|
52
|
+
def frame(height, width)
|
|
53
|
+
lines = begin
|
|
54
|
+
body(Snapshot.take(@config))
|
|
55
|
+
rescue => e
|
|
56
|
+
["", "Can't read from Redis: #{e.class}: #{e}", "Retrying every #{@interval}s."]
|
|
57
|
+
end
|
|
58
|
+
[header(width), tabs, *lines].first(height).map { |line| fit(line, width) }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Acts on a key press; false means quit.
|
|
62
|
+
def key(char)
|
|
63
|
+
case char
|
|
64
|
+
when *QUIT then return false
|
|
65
|
+
when "1", "2", "3" then @view = Integer(char) - 1
|
|
66
|
+
when "\t" then @view = (@view + 1) % VIEWS.size
|
|
67
|
+
end
|
|
68
|
+
true
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def loop_frames
|
|
74
|
+
loop do
|
|
75
|
+
height, width = size
|
|
76
|
+
@output.write("\e[H", frame(height, width).map { |l| "#{l}\e[K" }.join("\r\n"), "\e[J")
|
|
77
|
+
@output.flush
|
|
78
|
+
next unless @input.wait_readable(@interval)
|
|
79
|
+
|
|
80
|
+
chars = @input.read_nonblock(KEY_BUFFER, exception: false)
|
|
81
|
+
break if chars.nil? # the end of input
|
|
82
|
+
next if chars == :wait_readable
|
|
83
|
+
break unless chars.each_char.all? { |c| key(c) }
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def size
|
|
88
|
+
height, width = @output.winsize
|
|
89
|
+
(height.positive? && width.positive?) ? [height, width] : DEFAULT_SIZE
|
|
90
|
+
rescue SystemCallError, NoMethodError
|
|
91
|
+
DEFAULT_SIZE
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def header(width)
|
|
95
|
+
left = "Robin #{VERSION}"
|
|
96
|
+
right = Time.at(Rational(Robin.now_ms, 1000)).strftime("%H:%M:%S")
|
|
97
|
+
left + " " * [width - left.size - right.size, 1].max + right
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def tabs
|
|
101
|
+
views = VIEWS.each_with_index.map { |name, i| (i == @view) ? "[#{i + 1} #{name}]" : " #{i + 1} #{name} " }
|
|
102
|
+
"#{views.join(" ")} tab next · r refresh · q quit"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def body(snap)
|
|
106
|
+
case @view
|
|
107
|
+
when 0 then overview(snap)
|
|
108
|
+
when 1 then processes(snap)
|
|
109
|
+
else busy(snap)
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def overview(snap)
|
|
114
|
+
s = snap.stats
|
|
115
|
+
busy = "#{s.workers_size}/#{snap.processes.sum { |p| p["concurrency"].to_i }}"
|
|
116
|
+
[
|
|
117
|
+
"",
|
|
118
|
+
pairs({"Processed" => number(s.processed), "Failed" => number(s.failed), "Busy" => busy,
|
|
119
|
+
"Enqueued" => number(s.enqueued)}),
|
|
120
|
+
pairs({"Scheduled" => number(s.scheduled_size), "Retries" => number(s.retry_size),
|
|
121
|
+
"Dead" => number(s.dead_size), "Processes" => number(snap.processes.size)}),
|
|
122
|
+
""
|
|
123
|
+
] + table(%w[QUEUE SIZE LATENCY], snap.queues.map do |name, size, latency, paused|
|
|
124
|
+
[paused ? "#{name} (paused)" : name, number(size), duration(latency)]
|
|
125
|
+
end, right: [1, 2], empty: "No queues")
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def processes(snap)
|
|
129
|
+
rows = snap.processes.map do |p|
|
|
130
|
+
flags = [("leader" if p.leader?), ("quiet" if p.quiet?)].compact.join(", ")
|
|
131
|
+
name = [p["hostname"], p["pid"]].compact.join(":")
|
|
132
|
+
name = "#{name} [#{p["tag"]}]" if p["tag"]
|
|
133
|
+
[name, "#{p.busy}/#{p["concurrency"].to_i}", megabytes(p.rss), since(snap.at, p.started_at),
|
|
134
|
+
since(snap.at, p.beat), flags, p["queues"].to_a.join(", ")]
|
|
135
|
+
end
|
|
136
|
+
[""] + table(%w[PROCESS BUSY RSS UP BEAT STATUS QUEUES], rows, right: [1, 2, 3, 4], empty: "No processes")
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def busy(snap)
|
|
140
|
+
rows = snap.work.sort_by { |_, _, work| work.fetch("run_at") }.map do |identity, tid, work|
|
|
141
|
+
job = JobRecord.new(work.fetch("payload"))
|
|
142
|
+
[identity, tid, work.fetch("queue"), job.display_class, since(snap.at, work.fetch("run_at")), Robin.dump_json(job.display_args)]
|
|
143
|
+
end
|
|
144
|
+
[""] + table(%w[PROCESS THREAD QUEUE JOB RUNNING ARGS], rows, right: [4], empty: "Nothing running")
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Columns padded to their widest cell; +right+ lists the columns
|
|
148
|
+
# aligned right (numbers).
|
|
149
|
+
def table(headers, rows, right:, empty:)
|
|
150
|
+
return [" #{empty}"] if rows.empty?
|
|
151
|
+
|
|
152
|
+
widths = headers.each_index.map { |i| [headers, *rows].map { |r| r.fetch(i).size }.max }
|
|
153
|
+
[headers, *rows].map do |row|
|
|
154
|
+
row.each_with_index.map { |cell, i| right.include?(i) ? cell.rjust(widths.fetch(i)) : cell.ljust(widths.fetch(i)) }.join(" ").rstrip
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def pairs(hash) = hash.map { |label, value| "#{label} #{value}" }.join(" ")
|
|
159
|
+
|
|
160
|
+
def fit(line, width) = (line.size > width) ? "#{line[0, width - 1]}…" : line
|
|
161
|
+
|
|
162
|
+
def number(n) = n.to_i.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
|
|
163
|
+
|
|
164
|
+
def megabytes(kb) = "#{(kb / 1024.0).round} MB"
|
|
165
|
+
|
|
166
|
+
# How long ago +ms+ (epoch milliseconds) was, before +now+.
|
|
167
|
+
def since(now, ms) = ms ? duration((now - ms) / 1000.0) : "-"
|
|
168
|
+
|
|
169
|
+
def duration(seconds)
|
|
170
|
+
seconds = [seconds.to_f, 0.0].max
|
|
171
|
+
if seconds < 60 then "#{seconds.round(1)}s"
|
|
172
|
+
elsif seconds < 3600 then "#{(seconds / 60).floor}m #{(seconds % 60).floor}s"
|
|
173
|
+
elsif seconds < 86_400 then "#{(seconds / 3600).floor}h #{(seconds % 3600 / 60).floor}m"
|
|
174
|
+
else "#{(seconds / 86_400).floor}d #{(seconds % 86_400 / 3600).floor}h"
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
end
|
|
178
|
+
end
|