robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
|
@@ -0,0 +1,658 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Runs a Robin server: worker threads for every capsule, the scheduler,
|
|
7
|
+
# and a heartbeat thread that publishes process state, flushes metrics,
|
|
8
|
+
# takes part in leader election and reacts to remote control signals
|
|
9
|
+
# sent from the web UI.
|
|
10
|
+
class Launcher
|
|
11
|
+
# How long a stopping:<identity> marker outlives the shutdown timeout:
|
|
12
|
+
# long enough for the process's heartbeat to expire and the leader to
|
|
13
|
+
# recover its jobs if it's killed before it finishes stopping.
|
|
14
|
+
STOPPING_MARGIN = 3600
|
|
15
|
+
|
|
16
|
+
# Seconds interrupted jobs get to wind down, after the shutdown timeout.
|
|
17
|
+
HARD_SHUTDOWN_GRACE = 3
|
|
18
|
+
|
|
19
|
+
# Heartbeat round trips slower than this (microseconds) get a warning.
|
|
20
|
+
SLOW_REDIS_US = 50_000
|
|
21
|
+
|
|
22
|
+
# Seconds without a job finishing before an idle process collects its
|
|
23
|
+
# garbage (see Config#idle_gc).
|
|
24
|
+
IDLE_GC_AFTER = 1
|
|
25
|
+
# GVL contention (a percentage) that, kept up for GVL_HIGH_BEATS
|
|
26
|
+
# heartbeats in a row, is worth a warning.
|
|
27
|
+
GVL_HIGH = 50
|
|
28
|
+
GVL_HIGH_BEATS = 3
|
|
29
|
+
|
|
30
|
+
attr_reader :config, :identity, :managers, :collector, :leader, :started_at
|
|
31
|
+
attr_accessor :on_stop_request
|
|
32
|
+
|
|
33
|
+
def initialize(config, embedded: false)
|
|
34
|
+
@config = config
|
|
35
|
+
@embedded = embedded
|
|
36
|
+
@identity = "#{Robin.hostname}:#{Process.pid}:#{SecureRandom.hex(4)}"
|
|
37
|
+
@collector = Metrics::Collector.new
|
|
38
|
+
@leader = Leader.new(config, @identity)
|
|
39
|
+
@scheduler = Scheduler.new(config)
|
|
40
|
+
@quiet = false
|
|
41
|
+
@done = false
|
|
42
|
+
@last = {}
|
|
43
|
+
@managers = config.capsules.values.map { |cap| Manager.new(cap, self) }
|
|
44
|
+
# Fetchers with acknowledgements left waiting (see #send_acks).
|
|
45
|
+
@acks = Thread::Queue.new
|
|
46
|
+
@managers.each { |m| m.fetcher.acker = @acks }
|
|
47
|
+
@hooks = {leader: [], beat: []}
|
|
48
|
+
@suspects = []
|
|
49
|
+
@gvl = GVL.new
|
|
50
|
+
@gvl_high = 0
|
|
51
|
+
@wake_mutex = Mutex.new
|
|
52
|
+
@wake = ConditionVariable.new
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def quiet? = @quiet
|
|
56
|
+
|
|
57
|
+
# How many workers it runs, in every capsule.
|
|
58
|
+
def concurrency = @managers.sum(&:concurrency)
|
|
59
|
+
|
|
60
|
+
# Has capsule +name+ run +count+ workers from now on (see
|
|
61
|
+
# Manager#resize), growing the Redis pool to fit them. Returns false,
|
|
62
|
+
# changing nothing, when the process is quiet (its workers are
|
|
63
|
+
# stopping), or when the pool's size was set (:pool_size) and it can't
|
|
64
|
+
# serve them all.
|
|
65
|
+
def resize(count, capsule: "default")
|
|
66
|
+
raise ArgumentError, "a capsule needs a concurrency above 0, not #{count.inspect}" unless count.instance_of?(Integer) && count.positive?
|
|
67
|
+
|
|
68
|
+
manager = @managers.find { |m| m.capsule.name == capsule }
|
|
69
|
+
raise ArgumentError, "no capsule named #{capsule.inspect}" unless manager
|
|
70
|
+
|
|
71
|
+
if quiet?
|
|
72
|
+
config.logger.warn("not resizing capsule #{capsule.inspect} to #{count}: the process is quiet")
|
|
73
|
+
return false
|
|
74
|
+
end
|
|
75
|
+
before = manager.concurrency
|
|
76
|
+
workers = concurrency - before + count
|
|
77
|
+
unless config.fit_redis_pool(workers)
|
|
78
|
+
config.logger.warn("not resizing capsule #{capsule.inspect} to #{count}: #{workers} workers need a Redis pool of " \
|
|
79
|
+
"#{workers + 2} connections, but :pool_size is #{config.redis_pool.size}")
|
|
80
|
+
return false
|
|
81
|
+
end
|
|
82
|
+
manager.resize(count)
|
|
83
|
+
config.logger.info("capsule #{capsule.inspect} now runs #{count} workers (was #{before})")
|
|
84
|
+
Events.emit(config, "workers.resized", capsule: capsule, from: before, to: count)
|
|
85
|
+
true
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def stopping? = @done
|
|
89
|
+
|
|
90
|
+
# Housekeeping hooks run on every heartbeat (:beat) or only on the
|
|
91
|
+
# leader (:leader). Used by cron, batches and other subsystems.
|
|
92
|
+
def every_beat(&block) = @hooks.fetch(:beat) << block
|
|
93
|
+
|
|
94
|
+
def every_leader_beat(&block) = @hooks.fetch(:leader) << block
|
|
95
|
+
|
|
96
|
+
def run
|
|
97
|
+
# All of Robin, before any of its threads start: two threads that
|
|
98
|
+
# reach one autoload at once can find its constant missing on JRuby
|
|
99
|
+
# 10.0. (The CLI and a swarm load it already. An embedded server
|
|
100
|
+
# didn't.)
|
|
101
|
+
Robin.eager_load!
|
|
102
|
+
@started_at = Robin.now_ms
|
|
103
|
+
check!
|
|
104
|
+
config.freeze_capsules!
|
|
105
|
+
# First, so a port that's taken fails the boot before anything starts.
|
|
106
|
+
start_health_server
|
|
107
|
+
Warmup.run(config) if config.jit_warmup
|
|
108
|
+
Leader.register(config, @leader)
|
|
109
|
+
# Recover before the first heartbeat registers this process's own
|
|
110
|
+
# in-flight set, so recovery can't mistake it for a dead one.
|
|
111
|
+
safely("startup recovery") { Recovery.run(config, pool: config.housekeeping_pool) }
|
|
112
|
+
begin
|
|
113
|
+
# A failing startup hook stops the boot before any work starts.
|
|
114
|
+
config.fire_event(:startup, reraise: true)
|
|
115
|
+
rescue
|
|
116
|
+
Leader.unregister(config)
|
|
117
|
+
@health&.stop
|
|
118
|
+
raise
|
|
119
|
+
end
|
|
120
|
+
Periodic.attach(self) unless config.periodic_blocks.empty?
|
|
121
|
+
@gvl.start if config.gvl_monitoring
|
|
122
|
+
safely("heartbeat") { beat }
|
|
123
|
+
@managers.each(&:start)
|
|
124
|
+
@scheduler.start
|
|
125
|
+
@heartbeat = housekeeping_thread("robin.heartbeat") { beat }
|
|
126
|
+
@chores = housekeeping_thread("robin.leader") { leader_chores }
|
|
127
|
+
@acker = Thread.new { send_acks }
|
|
128
|
+
@acker.name = "robin.acks"
|
|
129
|
+
@acker.priority = 1 # outranks worker threads, like housekeeping
|
|
130
|
+
Systemd.notify("READY=1") unless @embedded
|
|
131
|
+
self
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Stop fetching new jobs; running jobs finish.
|
|
135
|
+
def quiet
|
|
136
|
+
return if @quiet
|
|
137
|
+
|
|
138
|
+
@quiet = true
|
|
139
|
+
config.logger.info("quieting: no new jobs will be started")
|
|
140
|
+
@managers.each(&:quiet)
|
|
141
|
+
config.fire_event(:quiet, reverse: true)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Graceful shutdown: quiet, wait up to config.timeout for running jobs,
|
|
145
|
+
# interrupt the rest and put their jobs back on the queue.
|
|
146
|
+
def stop
|
|
147
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + config.timeout
|
|
148
|
+
Systemd.notify("STOPPING=1") unless @embedded
|
|
149
|
+
safely("mark stopping") { mark_stopping }
|
|
150
|
+
quiet
|
|
151
|
+
@done = true
|
|
152
|
+
wake_up # housekeeping threads stop now
|
|
153
|
+
@scheduler.stop
|
|
154
|
+
# Interrupted jobs get one shared grace period to wind down.
|
|
155
|
+
@managers.each { |m| m.stop(deadline, deadline + HARD_SHUTDOWN_GRACE) }
|
|
156
|
+
safely("final metrics flush") { config.redis { |c| @collector.flush(config, c) } }
|
|
157
|
+
if config.shutdown_requeue
|
|
158
|
+
safely("requeue in-flight jobs") do
|
|
159
|
+
config.redis do |conn|
|
|
160
|
+
count = Recovery.drain(config, conn, inflight_key, count: false)
|
|
161
|
+
config.logger.info("pushed #{count} unfinished job(s) back to their queues") if count > 0
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
safely("unregister") do
|
|
166
|
+
config.redis do |conn|
|
|
167
|
+
@leader.resign(conn)
|
|
168
|
+
conn.pipelined do |p|
|
|
169
|
+
p.call("SREM", config.key("processes"), identity)
|
|
170
|
+
p.call("SREM", config.key("inflights"), inflight_key)
|
|
171
|
+
p.call("DEL", proc_key, "#{proc_key}:work", "#{proc_key}-signals", stopping_key)
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
Leader.unregister(config)
|
|
176
|
+
# Let a heartbeat or chore that's already running finish, so it can't
|
|
177
|
+
# re-register this process after it has unregistered.
|
|
178
|
+
@heartbeat&.join(1)
|
|
179
|
+
@chores&.join(1)
|
|
180
|
+
@acks << nil # the workers are done
|
|
181
|
+
@acker&.join(1)
|
|
182
|
+
@health&.stop
|
|
183
|
+
@gvl.stop
|
|
184
|
+
config.fire_event(:shutdown, reverse: true)
|
|
185
|
+
config.logger.info("bye!")
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# This process's health, as served by Robin::HealthServer: healthy
|
|
189
|
+
# while its heartbeat reaches Redis and it isn't shutting down. A quiet
|
|
190
|
+
# process is still healthy (it's finishing its jobs).
|
|
191
|
+
def health
|
|
192
|
+
age = @beat_at && (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @beat_at)
|
|
193
|
+
status = if @done then "stopping"
|
|
194
|
+
elsif age.nil? || age > config.process_ttl then "stale"
|
|
195
|
+
elsif @quiet then "quiet"
|
|
196
|
+
else "ok"
|
|
197
|
+
end
|
|
198
|
+
{"healthy" => %w[ok quiet].include?(status), "status" => status, "identity" => identity,
|
|
199
|
+
"heartbeat_age" => age&.round(3), "busy" => busy, "concurrency" => concurrency}
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def inflight_key = config.key("inflight", identity)
|
|
203
|
+
|
|
204
|
+
def proc_key = config.process_key(identity)
|
|
205
|
+
|
|
206
|
+
def stopping_key = config.key("stopping", identity)
|
|
207
|
+
|
|
208
|
+
# -- worker callbacks ----------------------------------------------------
|
|
209
|
+
|
|
210
|
+
# Records what a worker is running, for the heartbeat (and its
|
|
211
|
+
# in-flight key, for the sweep). Only the job's raw payload, exactly
|
|
212
|
+
# as fetched, is kept: the live payload hash is being changed by the
|
|
213
|
+
# job (and holds decrypted arguments).
|
|
214
|
+
# Each worker keeps the job it's running to itself (Processor#work),
|
|
215
|
+
# so starting and finishing jobs share no lock: with real threads
|
|
216
|
+
# (JRuby) a shared one would serialize every worker twice per job.
|
|
217
|
+
def work_started(processor, unit)
|
|
218
|
+
processor.work = [unit.queue, unit.raw, Robin.now_ms, unit.key]
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def work_finished(processor, payload, elapsed_ms, ok)
|
|
222
|
+
processor.work = nil
|
|
223
|
+
processor.finished_at = monotonic
|
|
224
|
+
processor.finished += 1
|
|
225
|
+
klass = Robin.display_class(payload)
|
|
226
|
+
@collector.record(klass, elapsed_ms, ok)
|
|
227
|
+
statsd = config.statsd
|
|
228
|
+
statsd.distribution("jobs.perform_dist", elapsed_ms, tags: {class: klass}) if ok && statsd&.distributions?
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def work_finished_cleanup(processor)
|
|
232
|
+
processor.work = nil
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def busy = running.size
|
|
236
|
+
|
|
237
|
+
# [processor, [queue, raw, run_at, in-flight key]] for every job
|
|
238
|
+
# running now.
|
|
239
|
+
def running
|
|
240
|
+
@managers.flat_map(&:processors).filter_map do |processor|
|
|
241
|
+
work = processor.work
|
|
242
|
+
[processor, work] if work
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def process_info
|
|
247
|
+
{
|
|
248
|
+
"identity" => identity, "hostname" => Robin.hostname, "pid" => Process.pid,
|
|
249
|
+
"tag" => config.tag, "labels" => config.labels, "started_at" => @started_at && Robin.epoch_seconds(@started_at),
|
|
250
|
+
"version" => VERSION, "ruby" => RUBY_DESCRIPTION, "env" => config.environment,
|
|
251
|
+
"embedded" => @embedded, "concurrency" => concurrency,
|
|
252
|
+
"queues" => config.capsules.values.flat_map(&:queues).uniq,
|
|
253
|
+
"weights" => config.capsules.values.map(&:weights),
|
|
254
|
+
"capsules" => @managers.to_h do |m|
|
|
255
|
+
c = m.capsule
|
|
256
|
+
[c.name, {"concurrency" => m.concurrency, "mode" => c.strict? ? "strict" : "weighted", "weights" => c.weights}]
|
|
257
|
+
end
|
|
258
|
+
}
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# One heartbeat, in one round trip: publish state, flush metrics, renew
|
|
262
|
+
# leadership, run housekeeping hooks and handle remote signals.
|
|
263
|
+
def beat
|
|
264
|
+
now = Robin.now_ms
|
|
265
|
+
work = running.map do |processor, (queue, raw, run_at)|
|
|
266
|
+
[processor.thread&.name || processor.object_id, work_json(queue, raw, run_at)]
|
|
267
|
+
end
|
|
268
|
+
held = held_keys
|
|
269
|
+
ttl = config.process_ttl
|
|
270
|
+
gvl = @gvl.sample
|
|
271
|
+
candidate = config.leader_candidate
|
|
272
|
+
sweep = due?(60, :sweep)
|
|
273
|
+
config.housekeeping_pool.with do |conn|
|
|
274
|
+
rtt_start = Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
|
|
275
|
+
at = {} #: Hash[Symbol, Integer]
|
|
276
|
+
metrics = nil #: Metrics::drained?
|
|
277
|
+
replies = begin
|
|
278
|
+
conn.multi do |tx|
|
|
279
|
+
tx.call("EXISTS", proc_key)
|
|
280
|
+
tx.call("SADD", config.key("processes"), identity)
|
|
281
|
+
tx.call("SADD", config.key("inflights"), inflight_key)
|
|
282
|
+
# Recovery unregisters the in-flight keys of a process whose
|
|
283
|
+
# heartbeat lapsed; they're registered again as soon as it's back.
|
|
284
|
+
tx.call("SADD", inflight_key, *held) unless held.empty?
|
|
285
|
+
# The round trip time is the last heartbeat's: this one's is
|
|
286
|
+
# only known once it's back.
|
|
287
|
+
tx.call("HSET", proc_key, "info", Robin.dump_json(process_info), "beat", Robin.epoch_seconds(now),
|
|
288
|
+
"busy", work.size, "concurrency", concurrency, "quiet", @quiet, "rss", rss_kb,
|
|
289
|
+
# (Steep doesn't take splatting nil, which adds no arguments.)
|
|
290
|
+
"leader", @leader.leader? ? "1" : "0", *(["rtt_us", @rtt] if @rtt), *(["gvl", gvl] if gvl)) # steep:ignore ArgumentTypeMismatch
|
|
291
|
+
tx.call("EXPIRE", proc_key, ttl)
|
|
292
|
+
tx.call("DEL", "#{proc_key}:work")
|
|
293
|
+
tx.call("HSET", "#{proc_key}:work", *work.flatten) unless work.empty?
|
|
294
|
+
tx.call("EXPIRE", "#{proc_key}:work", ttl)
|
|
295
|
+
at[:signals] = tx.commands.size
|
|
296
|
+
tx.call("LRANGE", "#{proc_key}-signals", 0, -1)
|
|
297
|
+
tx.call("DEL", "#{proc_key}-signals")
|
|
298
|
+
tx.call(*Cancellation.listing(config))
|
|
299
|
+
tx.call("SMEMBERS", config.key("paused"))
|
|
300
|
+
tx.call(*JobClass.listing(config))
|
|
301
|
+
at[:lease] = tx.commands.size
|
|
302
|
+
@leader.lease(tx) if candidate
|
|
303
|
+
at[:sweep] = tx.commands.size
|
|
304
|
+
tx.call("SMEMBERS", inflight_key) if sweep
|
|
305
|
+
metrics = @collector.queue(config, tx)
|
|
306
|
+
end
|
|
307
|
+
rescue
|
|
308
|
+
@collector.restore(metrics) if metrics
|
|
309
|
+
raise
|
|
310
|
+
end
|
|
311
|
+
# (The transaction went through, so metrics is set.)
|
|
312
|
+
@collector.flushed(config, metrics) # steep:ignore ArgumentTypeMismatch
|
|
313
|
+
warn_about_lapse if @beaten && replies.first.zero?
|
|
314
|
+
@beaten = true
|
|
315
|
+
@beat_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
316
|
+
signals, _, cancelled, paused, paused_classes = replies.drop(at.fetch(:signals)) #: [Array[String], Integer, Array[String], Array[String], Array[String]]
|
|
317
|
+
@rtt = Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond) - rtt_start
|
|
318
|
+
warn_about_slow_redis(@rtt) if @rtt > SLOW_REDIS_US
|
|
319
|
+
check_gvl(gvl) if gvl
|
|
320
|
+
|
|
321
|
+
@leader.settle(replies.fetch(at.fetch(:lease))) if candidate
|
|
322
|
+
Cancellation.store(config, cancelled)
|
|
323
|
+
JobClass.store(config, paused_classes)
|
|
324
|
+
hand_back_stale(conn, paused)
|
|
325
|
+
@hooks.fetch(:beat).each { |h| safely("beat hook") { h.call(conn) } }
|
|
326
|
+
safely("sweep") { sweep_inflight(conn, replies.fetch(at.fetch(:sweep))) } if sweep
|
|
327
|
+
signals.reverse_each { |sig| handle_signal(sig) } # pushed onto the head: oldest last
|
|
328
|
+
end
|
|
329
|
+
check_memory
|
|
330
|
+
unless @embedded
|
|
331
|
+
collect_idle_garbage
|
|
332
|
+
Process.setproctitle(title)
|
|
333
|
+
Systemd.notify("WATCHDOG=1") if Systemd.watchdog?
|
|
334
|
+
end
|
|
335
|
+
config.fire_event(:heartbeat)
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
# Runs a full GC once the process has gone idle (see Config#idle_gc):
|
|
339
|
+
# no job running, none finished for IDLE_GC_AFTER seconds, and at
|
|
340
|
+
# least config.idle_gc finished since the last time. Each worker
|
|
341
|
+
# counts its own jobs (Processor#finished), so finishing one takes no
|
|
342
|
+
# lock; the counts at the last collection are kept by worker.
|
|
343
|
+
def collect_idle_garbage
|
|
344
|
+
least = config.idle_gc
|
|
345
|
+
return unless least && running.empty?
|
|
346
|
+
|
|
347
|
+
processors = @managers.flat_map(&:processors)
|
|
348
|
+
last = processors.filter_map(&:finished_at).max
|
|
349
|
+
return unless last && monotonic - last >= IDLE_GC_AFTER
|
|
350
|
+
|
|
351
|
+
counted = @collected_counts || {}
|
|
352
|
+
return if processors.sum { |p| p.finished - counted.fetch(p, 0) } < least
|
|
353
|
+
|
|
354
|
+
collect_garbage
|
|
355
|
+
@collected_counts = processors.to_h { |p| [p, p.finished] }
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
def collect_garbage = GC.start
|
|
359
|
+
|
|
360
|
+
# Puts back the jobs every capsule fetched for its workers that are
|
|
361
|
+
# from a queue paused since, or that no worker took in time (see
|
|
362
|
+
# Fetcher#stale), in one round trip.
|
|
363
|
+
def hand_back_stale(conn, paused)
|
|
364
|
+
stale = @managers.map { |m| [m.fetcher, m.fetcher.stale(paused)] } #: Array[[Fetcher, Array[Fetcher::UnitOfWork]]]
|
|
365
|
+
stale.reject! { |_fetcher, units| units.empty? }
|
|
366
|
+
return if stale.empty?
|
|
367
|
+
|
|
368
|
+
conn.multi { |tx| stale.each { |fetcher, units| fetcher.hand_back(tx, units) } }
|
|
369
|
+
stale.each { |fetcher, units| fetcher.freed(units) }
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# The process title (as ps shows it) of a standalone server.
|
|
373
|
+
def title
|
|
374
|
+
state = if @done then " stopping"
|
|
375
|
+
elsif @quiet then " quiet"
|
|
376
|
+
end
|
|
377
|
+
"robin #{VERSION} #{config.tag} [#{busy} of #{concurrency} busy]#{state}"
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# Remote signals are named like the Unix signals that do the same.
|
|
381
|
+
def handle_signal(signal)
|
|
382
|
+
case signal
|
|
383
|
+
when "TSTP" then quiet
|
|
384
|
+
when "TERM" then request_stop
|
|
385
|
+
when "TTIN" then dump_threads
|
|
386
|
+
# (The match sets $1 and $2.)
|
|
387
|
+
when /\ARESIZE (\d+) (.+)\z/m then safely("resize") { resize(Integer($1, 10), capsule: $2) } # steep:ignore ArgumentTypeMismatch
|
|
388
|
+
else config.logger.warn("ignoring unknown remote signal #{signal.inspect}")
|
|
389
|
+
end
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
def request_stop
|
|
393
|
+
if on_stop_request
|
|
394
|
+
on_stop_request.call
|
|
395
|
+
else
|
|
396
|
+
Thread.new { stop }
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
def dump_threads
|
|
401
|
+
Thread.list.each do |t|
|
|
402
|
+
config.logger.warn("Thread #{t.name || t.object_id} (#{t.status})")
|
|
403
|
+
config.logger.warn((t.backtrace || ["<no backtrace>"]).join("\n"))
|
|
404
|
+
end
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
private
|
|
408
|
+
|
|
409
|
+
# {"queue", "run_at" (epoch seconds), "payload" (the job's JSON, as
|
|
410
|
+
# fetched)} as JSON.
|
|
411
|
+
def work_json(queue, raw, run_at)
|
|
412
|
+
Robin.dump_json({"queue" => queue, "payload" => raw, "run_at" => Robin.epoch_seconds(run_at)})
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
MINIMUM_REDIS = [6, 2].freeze
|
|
416
|
+
|
|
417
|
+
# Refuses settings and servers Robin can't run correctly with, and
|
|
418
|
+
# warns about ones that risk losing data.
|
|
419
|
+
def check!
|
|
420
|
+
config.capsules.each_value do |capsule|
|
|
421
|
+
raise ArgumentError, "capsule #{capsule.name.inspect} needs a concurrency above 0" unless capsule.concurrency.positive?
|
|
422
|
+
end
|
|
423
|
+
raise ArgumentError, "the shutdown timeout must be above 0 seconds" unless config.timeout.positive?
|
|
424
|
+
raise ArgumentError, "the fetch timeout must be above 0 seconds" unless config.fetch_timeout.positive?
|
|
425
|
+
|
|
426
|
+
needed = config.total_concurrency + 2
|
|
427
|
+
if config.redis_pool.size < needed
|
|
428
|
+
raise ArgumentError, "the Redis pool has #{config.redis_pool.size} connections but #{config.total_concurrency} " \
|
|
429
|
+
"worker threads need at least #{needed}; raise :pool_size or leave it unset"
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
info, conn = config.redis { |c| [c.call("INFO"), c] }
|
|
433
|
+
version = info[/^redis_version:(\S+)/, 1]
|
|
434
|
+
if (version.split(".").map(&:to_i) <=> MINIMUM_REDIS).negative?
|
|
435
|
+
raise ArgumentError, "Robin needs Redis #{MINIMUM_REDIS.join(".")} or later; this server runs #{version}"
|
|
436
|
+
end
|
|
437
|
+
config.logger.info(connected(conn, version))
|
|
438
|
+
|
|
439
|
+
policy = info[/^maxmemory_policy:(\S+)/, 1]
|
|
440
|
+
return if policy.nil? || policy == "noeviction"
|
|
441
|
+
|
|
442
|
+
config.logger.warn("Redis maxmemory-policy is #{policy}; set it to noeviction, or Redis may evict " \
|
|
443
|
+
"jobs and process heartbeats when memory runs low")
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
# Where this process's jobs are, so connecting to the wrong Redis (the
|
|
447
|
+
# default 127.0.0.1:6379, say) is plain to see: the server (never its
|
|
448
|
+
# password) and the key prefixes.
|
|
449
|
+
def connected(conn, version)
|
|
450
|
+
tls = " over TLS" if conn.options.fetch(:ssl)
|
|
451
|
+
own = config.internal_key_prefix
|
|
452
|
+
own = (", Robin's own under #{own.inspect}" unless own == config.key_prefix)
|
|
453
|
+
"Redis #{version} at #{conn.id}#{tls}; keys under #{config.key_prefix.inspect}#{own}"
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
# Swarm children listen on the configured port plus their index.
|
|
457
|
+
def start_health_server
|
|
458
|
+
return unless config.health_port
|
|
459
|
+
|
|
460
|
+
port = config.health_port + Integer(ENV.fetch("ROBIN_SWARM_INDEX", 0))
|
|
461
|
+
@health = HealthServer.new(self, port: port, bind: config.health_bind)
|
|
462
|
+
config.logger.info("health check listening on #{config.health_bind}:#{@health.port}")
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
# Jobs interrupted by this shutdown aren't poison pills, even if the
|
|
466
|
+
# process is killed before it can requeue them itself.
|
|
467
|
+
def mark_stopping
|
|
468
|
+
config.redis { |conn| conn.call("SET", stopping_key, "1", "EX", (config.timeout + STOPPING_MARGIN).ceil) }
|
|
469
|
+
end
|
|
470
|
+
|
|
471
|
+
# A thread that runs the block every heartbeat interval until stopping.
|
|
472
|
+
# It outranks worker threads, so busy jobs can't starve it.
|
|
473
|
+
def housekeeping_thread(name)
|
|
474
|
+
Thread.new do
|
|
475
|
+
Thread.current.name = name
|
|
476
|
+
Thread.current.priority = 1
|
|
477
|
+
until @done
|
|
478
|
+
nap(config.heartbeat_interval)
|
|
479
|
+
safely(name.delete_prefix("robin.")) { yield } unless @done
|
|
480
|
+
end
|
|
481
|
+
end
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
# Sleeps up to +seconds+, or until #wake_up (called by #stop).
|
|
485
|
+
# mutant:disable -- the lock and the @done check only close a lost-wakeup
|
|
486
|
+
# race (a stop landing between the loop's check and the wait) that tests
|
|
487
|
+
# can't provoke deterministically; test_stop_wakes_sleeping_housekeeping_threads
|
|
488
|
+
# covers the behavior.
|
|
489
|
+
def nap(seconds)
|
|
490
|
+
@wake_mutex.synchronize { @wake.wait(@wake_mutex, seconds) unless @done }
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
# mutant:disable -- see #nap
|
|
494
|
+
def wake_up
|
|
495
|
+
@wake_mutex.synchronize { @wake.broadcast }
|
|
496
|
+
end
|
|
497
|
+
|
|
498
|
+
# Sends the acknowledgements fetchers leave waiting, ACK_WAIT after
|
|
499
|
+
# they put themselves on @acks (see Fetcher#acker), until a nil comes
|
|
500
|
+
# off it. Only the first failure of a run of them is reported (workers
|
|
501
|
+
# report a Redis outage too); those acknowledgements wait again.
|
|
502
|
+
# mutant:disable -- Thread::Queue#shift is an alias of #pop.
|
|
503
|
+
def send_acks
|
|
504
|
+
failing = false
|
|
505
|
+
loop do
|
|
506
|
+
fetchers = [@acks.pop]
|
|
507
|
+
break unless fetchers.first
|
|
508
|
+
|
|
509
|
+
sleep Fetcher::ACK_WAIT
|
|
510
|
+
fetchers << @acks.pop until @acks.empty? # this thread is the only taker
|
|
511
|
+
fetchers.compact.each do |fetcher|
|
|
512
|
+
fetcher.send_acks
|
|
513
|
+
failing = false
|
|
514
|
+
rescue => e
|
|
515
|
+
config.handle_exception(e, {context: "acknowledging jobs"}) unless failing
|
|
516
|
+
failing = true
|
|
517
|
+
end
|
|
518
|
+
break if fetchers.include?(nil)
|
|
519
|
+
end
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
# Cluster-wide duties of the leader, on their own thread so slow ones
|
|
523
|
+
# (recovery after a big crash) never delay this process's heartbeat.
|
|
524
|
+
def leader_chores
|
|
525
|
+
return unless @leader.leader?
|
|
526
|
+
|
|
527
|
+
config.housekeeping_pool.with do |conn|
|
|
528
|
+
every(60, :sample_queues) { Metrics.sample_queues(config, conn) }
|
|
529
|
+
every(30, :recovery) { Recovery.run(config, pool: config.housekeeping_pool) }
|
|
530
|
+
every(60, :prune) { prune_processes(conn) }
|
|
531
|
+
@hooks.fetch(:leader).each { |h| safely("leader hook") { h.call(conn) } }
|
|
532
|
+
end
|
|
533
|
+
end
|
|
534
|
+
|
|
535
|
+
# The heartbeat key had expired: for a while this process looked dead, so
|
|
536
|
+
# the leader may have recovered (and another process re-run) jobs that
|
|
537
|
+
# were still running here.
|
|
538
|
+
# At most every 10 minutes: a slow Redis slows every job down.
|
|
539
|
+
def warn_about_slow_redis(rtt_us)
|
|
540
|
+
every(600, :slow_redis) do
|
|
541
|
+
config.logger.warn(format("Redis round trips take %.1fms; jobs will be slow (is Redis far away, or overloaded?)", rtt_us / 1000.0))
|
|
542
|
+
Events.emit(config, "redis.slow", rtt_us: rtt_us)
|
|
543
|
+
end
|
|
544
|
+
end
|
|
545
|
+
|
|
546
|
+
# Contention is reported every heartbeat (StatsD's gvl.contention);
|
|
547
|
+
# contention kept high warns, at most every 10 minutes.
|
|
548
|
+
def check_gvl(percent)
|
|
549
|
+
config.statsd&.gauge("gvl.contention", percent)
|
|
550
|
+
@gvl_high = (percent >= GVL_HIGH) ? @gvl_high + 1 : 0
|
|
551
|
+
return if @gvl_high < GVL_HIGH_BEATS
|
|
552
|
+
|
|
553
|
+
every(600, :gvl) do
|
|
554
|
+
config.logger.warn("GVL contention is #{percent}%: threads spend most of their time waiting for each other; " \
|
|
555
|
+
"fewer threads per process (and more processes) would run CPU-heavy jobs faster")
|
|
556
|
+
Events.emit(config, "gvl.contended", percent: percent)
|
|
557
|
+
end
|
|
558
|
+
end
|
|
559
|
+
|
|
560
|
+
def warn_about_lapse
|
|
561
|
+
config.logger.warn("heartbeat lapsed (more than #{config.process_ttl}s without one); jobs running here " \
|
|
562
|
+
"may have been recovered and run again elsewhere. Is the process starved of CPU or Redis connections?")
|
|
563
|
+
Events.emit(config, "heartbeat.lapsed")
|
|
564
|
+
end
|
|
565
|
+
|
|
566
|
+
# The in-flight keys this process's workers hold: those of running
|
|
567
|
+
# jobs, of finished jobs whose acknowledgement waits for a later
|
|
568
|
+
# fetch, of jobs fetched for a worker that hasn't asked yet, and free
|
|
569
|
+
# ones. Running jobs are read first, so one finishing meanwhile is seen
|
|
570
|
+
# either way.
|
|
571
|
+
def held_keys
|
|
572
|
+
held = running.map { |_processor, work| work.fetch(3) } #: Array[String?]
|
|
573
|
+
held.concat(@managers.flat_map { |m| m.processors.map(&:unacked) + m.fetcher.held })
|
|
574
|
+
held.compact
|
|
575
|
+
end
|
|
576
|
+
|
|
577
|
+
# Puts back jobs that sit in this process's in-flight keys without any
|
|
578
|
+
# worker holding them (e.g. one died after the job finished but before
|
|
579
|
+
# acknowledging it, or a round trip failed after filling them), and
|
|
580
|
+
# unregisters keys left empty that way. A key must be unheld, with the
|
|
581
|
+
# same job (or none) in it, on two sweeps in a row, so one that's just
|
|
582
|
+
# being fetched or acknowledged (or was used again since) is never
|
|
583
|
+
# taken.
|
|
584
|
+
# +registered+ are the keys registered for this process, as the
|
|
585
|
+
# heartbeat read them.
|
|
586
|
+
def sweep_inflight(conn, registered)
|
|
587
|
+
unheld = registered - held_keys
|
|
588
|
+
sightings = unheld.zip(conn.pipelined { |p| unheld.each { |key| p.call("LINDEX", key, 0) } })
|
|
589
|
+
orphans = sightings & @suspects
|
|
590
|
+
@suspects = sightings - orphans
|
|
591
|
+
orphans.select! { |key, _raw| @managers.all? { |m| m.fetcher.forget(key) } }
|
|
592
|
+
requeued = conn.pipelined do |p|
|
|
593
|
+
orphans.each do |key, raw|
|
|
594
|
+
# An empty key (raw nil) is only unregistered.
|
|
595
|
+
queue = raw ? Robin.load_json(raw).fetch("queue") : ""
|
|
596
|
+
p.evalsha(Scripts::REQUEUE_ONE, [key, config.key("queue", queue), config.key("queues"), inflight_key], [raw.to_s, queue])
|
|
597
|
+
end
|
|
598
|
+
end.sum
|
|
599
|
+
return unless requeued.positive?
|
|
600
|
+
|
|
601
|
+
config.logger.warn("put back #{requeued} orphaned in-flight job(s)")
|
|
602
|
+
config.statsd&.increment("jobs.orphaned", requeued)
|
|
603
|
+
Events.emit(config, "jobs.orphaned", count: requeued)
|
|
604
|
+
end
|
|
605
|
+
|
|
606
|
+
def every(seconds, name)
|
|
607
|
+
safely(name.to_s) { yield } if due?(seconds, name)
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
# Whether +seconds+ have passed since +name+ was last due (or it never
|
|
611
|
+
# was); it's due again +seconds+ from now if so.
|
|
612
|
+
def due?(seconds, name)
|
|
613
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
614
|
+
last = @last[name]
|
|
615
|
+
return false if last && now - last < seconds
|
|
616
|
+
|
|
617
|
+
@last[name] = now
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
# mutant:disable -- Connection#evalsha splats ARGV, so passing the
|
|
621
|
+
# prefix bare or wrapped in an Array is identical.
|
|
622
|
+
def prune_processes(conn) = conn.evalsha(Scripts::PRUNE, [config.key("processes")], [config.key_prefix])
|
|
623
|
+
|
|
624
|
+
def check_memory
|
|
625
|
+
limit = config.memory_limit_mb
|
|
626
|
+
return unless limit && !@done
|
|
627
|
+
|
|
628
|
+
rss_mb = rss_kb / 1024
|
|
629
|
+
return if rss_mb < limit
|
|
630
|
+
|
|
631
|
+
config.logger.warn("RSS #{rss_mb}MB exceeds memory_limit_mb #{limit}MB; restarting gracefully")
|
|
632
|
+
Events.emit(config, "memory.exceeded", rss_mb: rss_mb, limit_mb: limit)
|
|
633
|
+
request_stop
|
|
634
|
+
end
|
|
635
|
+
|
|
636
|
+
def rss_kb
|
|
637
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
638
|
+
return @rss if @rss && now - @rss_at < 15
|
|
639
|
+
|
|
640
|
+
@rss_at = now
|
|
641
|
+
@rss = if File.readable?("/proc/self/status")
|
|
642
|
+
Integer(File.read("/proc/self/status")[/^VmRSS:\s*(\d+)/, 1])
|
|
643
|
+
else
|
|
644
|
+
Integer(`ps -o rss= -p #{Process.pid}`)
|
|
645
|
+
end
|
|
646
|
+
rescue
|
|
647
|
+
@rss = 0
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
def safely(what)
|
|
651
|
+
yield
|
|
652
|
+
rescue => e
|
|
653
|
+
config.handle_exception(e, {context: what})
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
657
|
+
end
|
|
658
|
+
end
|