robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "zlib"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Profiles single job runs with Vernier, a sampling profiler (add
|
|
7
|
+
# `gem "vernier"`; CRuby only), to find out why a job is slow:
|
|
8
|
+
#
|
|
9
|
+
# ReportJob.set(profile: true).perform_async(42)
|
|
10
|
+
# ReportJob.set(profile: "big export").perform_async(43) # with a label
|
|
11
|
+
#
|
|
12
|
+
# The profile, in the Firefox Profiler format, is kept in Redis for a
|
|
13
|
+
# week (the newest KEEP of them). The dashboard lists them and opens them
|
|
14
|
+
# in https://profiler.firefox.com; from Ruby:
|
|
15
|
+
#
|
|
16
|
+
# Robin::Profiler.list # newest first, without the data
|
|
17
|
+
# Robin::Profiler.data(jid) # the profile JSON, or nil
|
|
18
|
+
#
|
|
19
|
+
# A process profiles one job at a time; others asking meanwhile run
|
|
20
|
+
# unprofiled. Failed runs are profiled too.
|
|
21
|
+
module Profiler
|
|
22
|
+
TTL = 7 * 86_400
|
|
23
|
+
KEEP = 100
|
|
24
|
+
MAX_BYTES = 16 * 1024 * 1024 # compressed; bigger profiles are dropped
|
|
25
|
+
LOCK = Mutex.new
|
|
26
|
+
|
|
27
|
+
# Loads Vernier once, warning when it can't.
|
|
28
|
+
def self.available?(config)
|
|
29
|
+
return @available unless @available.nil?
|
|
30
|
+
|
|
31
|
+
@available = begin
|
|
32
|
+
require "vernier"
|
|
33
|
+
true
|
|
34
|
+
rescue LoadError
|
|
35
|
+
config.logger.warn("profiling a job needs the vernier gem (CRuby); running it unprofiled")
|
|
36
|
+
false
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Runs the block under the profiler and stores the result. Returns the
|
|
41
|
+
# block's value.
|
|
42
|
+
def self.profile(config, payload)
|
|
43
|
+
collector = start(config)
|
|
44
|
+
return yield unless collector
|
|
45
|
+
|
|
46
|
+
started = Robin.now_ms
|
|
47
|
+
ok = false
|
|
48
|
+
begin
|
|
49
|
+
result = yield
|
|
50
|
+
ok = true
|
|
51
|
+
result
|
|
52
|
+
ensure
|
|
53
|
+
record(config, payload, collector, started, ok)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# A running collector, or nil if one can't start (e.g. the app is
|
|
58
|
+
# profiling something else).
|
|
59
|
+
def self.start(config)
|
|
60
|
+
Vernier::Collector.new(:wall, {}).tap(&:start)
|
|
61
|
+
rescue => e
|
|
62
|
+
config.logger.warn("couldn't start the profiler (#{e.class}: #{e.message}); running the job unprofiled")
|
|
63
|
+
nil
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def self.record(config, payload, collector, started, ok)
|
|
67
|
+
duration = Robin.now_ms - started
|
|
68
|
+
data = Zlib.gzip(collector.stop.to_gecko)
|
|
69
|
+
jid = payload.fetch("jid")
|
|
70
|
+
if data.bytesize > MAX_BYTES
|
|
71
|
+
config.logger.warn("profile of #{payload.fetch("class")} jid=#{jid} is too big to keep (#{data.bytesize} bytes)")
|
|
72
|
+
return
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
label = payload.fetch("profile")
|
|
76
|
+
entry = {"jid" => jid, "class" => Robin.display_class(payload), "queue" => payload["queue"],
|
|
77
|
+
"label" => label.equal?(true) ? nil : label.to_s, "started_at" => started,
|
|
78
|
+
"duration_ms" => duration, "ok" => ok, "bytes" => data.bytesize}
|
|
79
|
+
config.redis do |conn|
|
|
80
|
+
conn.multi do |tx|
|
|
81
|
+
tx.call("SET", config.key("profile", jid), data, "EX", TTL)
|
|
82
|
+
tx.call("ZADD", config.key("profiles"), started, Robin.dump_json(entry))
|
|
83
|
+
tx.call("ZREMRANGEBYSCORE", config.key("profiles"), "-inf", "(#{started - TTL * 1000}")
|
|
84
|
+
tx.call("ZREMRANGEBYRANK", config.key("profiles"), 0, -KEEP - 1)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
rescue => e
|
|
88
|
+
config.handle_exception(e, {context: "storing a job profile", job: payload})
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Stored profiles, newest first (without their data).
|
|
92
|
+
def self.list(config: Robin.config)
|
|
93
|
+
entries(config).map(&:last)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The profile JSON for +jid+, or nil once it has expired.
|
|
97
|
+
def self.data(jid, config: Robin.config)
|
|
98
|
+
data = config.redis { |c| c.call("GET", config.key("profile", jid)) }
|
|
99
|
+
data && Zlib.gunzip(data)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Deletes the profile of +jid+; true if there was one.
|
|
103
|
+
def self.delete(jid, config: Robin.config)
|
|
104
|
+
raws = entries(config).filter_map { |raw, entry| raw if entry.fetch("jid") == jid }
|
|
105
|
+
config.redis do |c|
|
|
106
|
+
c.multi do |tx|
|
|
107
|
+
tx.call("DEL", config.key("profile", jid))
|
|
108
|
+
tx.call("ZREM", config.key("profiles"), *raws) unless raws.empty?
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
!raws.empty?
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# [raw, parsed] pairs, newest first.
|
|
115
|
+
def self.entries(config)
|
|
116
|
+
config.redis { |c| c.call("ZRANGE", config.key("profiles"), 0, -1, "REV") }.map { |raw| [raw, Robin.load_json(raw)] }
|
|
117
|
+
end
|
|
118
|
+
private_class_method :entries
|
|
119
|
+
|
|
120
|
+
# Profiles jobs pushed with a "profile" option. Installed by default;
|
|
121
|
+
# a no-op for every other job.
|
|
122
|
+
class ServerMiddleware
|
|
123
|
+
include Middleware::ConfigAware
|
|
124
|
+
include Middleware::Stateless
|
|
125
|
+
|
|
126
|
+
def self.applies?(_job, payload, _queue) = payload.key?("profile")
|
|
127
|
+
|
|
128
|
+
def call(_job, payload, _queue, &)
|
|
129
|
+
return yield unless payload["profile"] && Profiler.available?(config)
|
|
130
|
+
return yield unless LOCK.try_lock
|
|
131
|
+
|
|
132
|
+
begin
|
|
133
|
+
Profiler.profile(config, payload, &)
|
|
134
|
+
ensure
|
|
135
|
+
LOCK.unlock
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Live progress reporting for long jobs, shown on the web UI's Busy page.
|
|
5
|
+
#
|
|
6
|
+
# def perform(ids)
|
|
7
|
+
# ids.each_with_index do |id, i|
|
|
8
|
+
# process(id)
|
|
9
|
+
# progress!((i + 1) * 100 / ids.size, "#{i + 1}/#{ids.size} rows")
|
|
10
|
+
# end
|
|
11
|
+
# end
|
|
12
|
+
module Progress
|
|
13
|
+
TTL = 86_400
|
|
14
|
+
|
|
15
|
+
class << self
|
|
16
|
+
def report(jid, percent, message = nil, meta = {}, config: Robin.config)
|
|
17
|
+
data = entry(percent, message, meta)
|
|
18
|
+
config.redis { |c| c.call(*set_command(config, jid, data)) }
|
|
19
|
+
data
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# What #report stores. A nil +percent+ is unknown (the Busy page
|
|
23
|
+
# shows the message, or meta's "iterations" count, without a bar).
|
|
24
|
+
def entry(percent, message = nil, meta = {})
|
|
25
|
+
data = {"pct" => percent&.to_f&.clamp(0, 100)&.round(1), "msg" => message, "at" => Robin.now_ms} #: entry
|
|
26
|
+
data["meta"] = meta unless meta.empty?
|
|
27
|
+
data
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The command #report sends, for callers that batch it with others.
|
|
31
|
+
def set_command(config, jid, data) = ["SET", config.key("progress", jid), Robin.dump_json(data), "EX", TTL]
|
|
32
|
+
|
|
33
|
+
def fetch(jid, config: Robin.config)
|
|
34
|
+
raw = config.redis { |c| c.call("GET", config.key("progress", jid)) }
|
|
35
|
+
raw && Robin.load_json(raw)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def fetch_many(jids, config: Robin.config)
|
|
39
|
+
return {} if jids.empty?
|
|
40
|
+
|
|
41
|
+
values = config.redis { |c| c.call("MGET", *jids.map { |j| config.key("progress", j) }) }
|
|
42
|
+
jids.zip(values).each_with_object({}) { |(j, v), h| h[j] = Robin.load_json(v) if v }
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "robin"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
module Rails
|
|
7
|
+
# Boots a Rails application directory for `bundle exec robin`: the
|
|
8
|
+
# CLI's first app loader (see Robin::CLI.app_loaders). Other
|
|
9
|
+
# directories are left to the next loader.
|
|
10
|
+
module AppLoader
|
|
11
|
+
# The Railtie is loaded first so it joins the app's initialization
|
|
12
|
+
# even when Bundler.require hasn't loaded robin (or loaded it before
|
|
13
|
+
# Rails).
|
|
14
|
+
def self.call(dir)
|
|
15
|
+
return false unless File.exist?(File.join(dir, "config", "application.rb"))
|
|
16
|
+
|
|
17
|
+
require "robin/rails"
|
|
18
|
+
require File.expand_path(File.join(dir, "config", "environment.rb"))
|
|
19
|
+
true
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_job"
|
|
4
|
+
require "active_job/base" # what ActiveJob::Arguments needs
|
|
5
|
+
|
|
6
|
+
module Robin
|
|
7
|
+
module Rails
|
|
8
|
+
# Carries ActiveSupport::CurrentAttributes from where a job is pushed
|
|
9
|
+
# into the job, so that e.g. Current.user and Current.tenant are set
|
|
10
|
+
# while it runs:
|
|
11
|
+
#
|
|
12
|
+
# config.robin.current_attributes = ["Current"] # class names
|
|
13
|
+
#
|
|
14
|
+
# Pushing saves each class's attributes in the job, the first listed
|
|
15
|
+
# class's under "cattr", the next under "cattr_1" and so on, serialized
|
|
16
|
+
# the way ActiveJob serializes arguments, so records travel as
|
|
17
|
+
# GlobalIDs. Running sets them around the job, inside the Rails
|
|
18
|
+
# executor (which resets them afterwards). Classes are looked up by
|
|
19
|
+
# name on every push and run, so code reloading keeps working.
|
|
20
|
+
#
|
|
21
|
+
# The attributes are stored in Redis as they are: unlike encrypted
|
|
22
|
+
# arguments, they're readable there and in the dashboard.
|
|
23
|
+
module CurrentAttributes
|
|
24
|
+
# The job field holding the attributes of the class at +index+ in
|
|
25
|
+
# the configured list.
|
|
26
|
+
def self.key_at(index) = index.zero? ? "cattr" : "cattr_#{index}"
|
|
27
|
+
|
|
28
|
+
# [field, class name] for each configured class.
|
|
29
|
+
def self.fields(names) = Array(names).each_with_index.map { |name, i| [key_at(i), name.to_s] }
|
|
30
|
+
|
|
31
|
+
# Saves the attributes into jobs being pushed.
|
|
32
|
+
class Save
|
|
33
|
+
include Robin::Middleware::Stateless
|
|
34
|
+
|
|
35
|
+
def initialize(names)
|
|
36
|
+
@fields = CurrentAttributes.fields(names)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def call(_job_class, payload, _queue, _redis)
|
|
40
|
+
@fields.each do |key, name|
|
|
41
|
+
next if payload.key?(key)
|
|
42
|
+
|
|
43
|
+
attributes = name.constantize.attributes
|
|
44
|
+
payload[key] = ::ActiveJob::Arguments.serialize(attributes) unless attributes.empty?
|
|
45
|
+
end
|
|
46
|
+
yield
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Sets the saved attributes while a job runs.
|
|
51
|
+
class Load
|
|
52
|
+
include Robin::Middleware::Stateless
|
|
53
|
+
|
|
54
|
+
def initialize(names)
|
|
55
|
+
@fields = CurrentAttributes.fields(names)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def call(_job, payload, _queue, &) = set(@fields.select { |key, _| payload.key?(key) }, payload, &)
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
# Nests one CurrentAttributes.set per class around the job. A class
|
|
63
|
+
# that no longer exists (renamed since the push) is skipped.
|
|
64
|
+
def set(fields, payload, &job)
|
|
65
|
+
return yield if fields.empty?
|
|
66
|
+
|
|
67
|
+
key, name = fields.fetch(0)
|
|
68
|
+
rest = fields.drop(1)
|
|
69
|
+
klass = name.safe_constantize
|
|
70
|
+
return set(rest, payload, &job) unless klass
|
|
71
|
+
|
|
72
|
+
# An attribute removed since the push is left out.
|
|
73
|
+
attributes = ::ActiveJob::Arguments.deserialize(payload.fetch(key)).to_h
|
|
74
|
+
attributes = attributes.select { |name, _| klass.method_defined?(:"#{name}=") }
|
|
75
|
+
klass.set(attributes) { set(rest, payload, &job) }
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def self.install(config, names)
|
|
80
|
+
names = Array(names)
|
|
81
|
+
return if names.empty?
|
|
82
|
+
|
|
83
|
+
config.client_middleware { |chain| chain.add(Save, names) }
|
|
84
|
+
config.server_middleware { |chain| chain.prepend(Load, names) }
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
module Rails
|
|
5
|
+
# A Robin error handler that reports to the Rails error reporter
|
|
6
|
+
# (Rails.error), so every service subscribed to it sees Robin's errors.
|
|
7
|
+
# A failed job is reported as unhandled with source "robin.job"; Robin's
|
|
8
|
+
# own errors (e.g. Redis unreachable while fetching) as handled with
|
|
9
|
+
# source "robin". The context names the job, never its arguments.
|
|
10
|
+
#
|
|
11
|
+
# Rails.error reports an exception only once, so an app handler that
|
|
12
|
+
# also calls Rails.error.report doesn't double-report.
|
|
13
|
+
class ErrorReporter
|
|
14
|
+
JOB_FAILED = "job raised"
|
|
15
|
+
|
|
16
|
+
def initialize(reporter)
|
|
17
|
+
@reporter = reporter
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def call(exception, context, _config)
|
|
21
|
+
job = context.fetch(:job, {})
|
|
22
|
+
failed = context[:context] == JOB_FAILED
|
|
23
|
+
details = {
|
|
24
|
+
robin: context[:context], queue: context.fetch(:queue, job["queue"]), jid: job["jid"],
|
|
25
|
+
job_class: Robin.display_class(job), retry_count: job["retry_count"], bid: job["bid"]
|
|
26
|
+
}
|
|
27
|
+
@reporter.report(exception, handled: !failed, source: failed ? "robin.job" : "robin", context: details.compact)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/notifications"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
module Rails
|
|
7
|
+
# Publishes Robin's events (see Robin::Events) through
|
|
8
|
+
# ActiveSupport::Notifications as "<event>.robin":
|
|
9
|
+
#
|
|
10
|
+
# ActiveSupport::Notifications.subscribe("job.expired.robin") do |event|
|
|
11
|
+
# Rails.logger.info("expired: #{event.payload[:job]["jid"]}")
|
|
12
|
+
# end
|
|
13
|
+
#
|
|
14
|
+
# The Railtie adds it to Robin.config.event_handlers unless
|
|
15
|
+
# config.robin.notifications is false.
|
|
16
|
+
module Notifications
|
|
17
|
+
def self.call(name, payload)
|
|
18
|
+
ActiveSupport::Notifications.instrument("#{name}.robin", payload)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails"
|
|
4
|
+
require "active_support/ordered_options"
|
|
5
|
+
|
|
6
|
+
module Robin
|
|
7
|
+
module Rails
|
|
8
|
+
# Wires Robin into a Rails app:
|
|
9
|
+
#
|
|
10
|
+
# * jobs run inside the Rails reloader and executor, so code reloading,
|
|
11
|
+
# query caches and connection checkin work as in a request
|
|
12
|
+
# * ActiveJob gets the :robin adapter and +robin_options+
|
|
13
|
+
# * Robin.config.environment follows Rails.env
|
|
14
|
+
# * outside the robin process (web, console), Robin logs to Rails.logger
|
|
15
|
+
# * errors are reported to Rails.error (see Robin::Rails::ErrorReporter)
|
|
16
|
+
# * failed jobs' backtraces go through Rails.backtrace_cleaner
|
|
17
|
+
# * what jobs log through Rails.logger is tagged with the job's class
|
|
18
|
+
# and jid (when the logger supports tags)
|
|
19
|
+
# * Robin's events are published through ActiveSupport::Notifications
|
|
20
|
+
# (see Robin::Rails::Notifications)
|
|
21
|
+
#
|
|
22
|
+
# Settings, e.g. in config/application.rb:
|
|
23
|
+
#
|
|
24
|
+
# config.robin.report_errors = false # default true
|
|
25
|
+
# config.robin.transactional_push = true # default false; see TransactionalPush
|
|
26
|
+
# config.robin.current_attributes = ["Current"] # see CurrentAttributes
|
|
27
|
+
# config.robin.notifications = false # default true
|
|
28
|
+
class Railtie < ::Rails::Railtie
|
|
29
|
+
config.robin = ActiveSupport::OrderedOptions.new
|
|
30
|
+
config.robin.report_errors = true
|
|
31
|
+
config.robin.transactional_push = false
|
|
32
|
+
config.robin.current_attributes = []
|
|
33
|
+
config.robin.notifications = true
|
|
34
|
+
|
|
35
|
+
# Before the app's initializers, so they can still override it.
|
|
36
|
+
initializer "robin.environment", before: :load_config_initializers do
|
|
37
|
+
Robin.config.environment = ::Rails.env.to_s
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Before ActiveJob's own hook selects the configured queue adapter.
|
|
41
|
+
initializer "robin.active_job", before: "active_job.set_configs" do
|
|
42
|
+
ActiveSupport.on_load(:active_job) { require "robin/active_job" }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
config.after_initialize { |app| Railtie.integrate(app) }
|
|
46
|
+
|
|
47
|
+
def self.integrate(app, config: Robin.config)
|
|
48
|
+
options = app.config.robin
|
|
49
|
+
config.job_wrapper ||= job_wrapper(app)
|
|
50
|
+
# (lambda, not ->: Steep splats a -> lambda's one Array parameter.)
|
|
51
|
+
config.backtrace_cleaner ||= lambda { |lines| ::Rails.backtrace_cleaner.clean(lines) }
|
|
52
|
+
config.logger = ::Rails.logger if !Robin.server? && ::Rails.logger
|
|
53
|
+
if options.report_errors && config.error_handlers.none?(ErrorReporter)
|
|
54
|
+
config.error_handlers << ErrorReporter.new(::Rails.error)
|
|
55
|
+
end
|
|
56
|
+
config.push_deferrer ||= TransactionalPush if options.transactional_push
|
|
57
|
+
if options.notifications && !config.event_handlers.include?(Notifications)
|
|
58
|
+
config.event_handlers << Notifications
|
|
59
|
+
end
|
|
60
|
+
CurrentAttributes.install(config, options.current_attributes)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Runs each job inside the app's reloader (and so its executor). A
|
|
64
|
+
# job's exception is carried out of the reloader and raised after it,
|
|
65
|
+
# so the executor doesn't report it: Robin does, once, with the job's
|
|
66
|
+
# context, and never for Robin::Discard.
|
|
67
|
+
def self.job_wrapper(app)
|
|
68
|
+
lambda do |payload, &job|
|
|
69
|
+
failure = nil #: Exception?
|
|
70
|
+
result = app.reloader.wrap do
|
|
71
|
+
tagged(payload, &job)
|
|
72
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
73
|
+
failure = e
|
|
74
|
+
end
|
|
75
|
+
failure ? raise(failure) : result
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def self.tagged(payload, &)
|
|
80
|
+
logger = ::Rails.logger
|
|
81
|
+
return yield unless logger.respond_to?(:tagged)
|
|
82
|
+
|
|
83
|
+
logger.tagged("#{Robin.display_class(payload)} jid=#{payload.fetch("jid")}", &)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
module Rails
|
|
5
|
+
# A Config#push_deferrer that holds jobs pushed inside Active Record
|
|
6
|
+
# transactions until all of them commit (rolled back transactions drop
|
|
7
|
+
# them), so a job never runs before the data it needs is visible. It
|
|
8
|
+
# needs ActiveRecord.after_all_transactions_commit (Rails 7.2+); without
|
|
9
|
+
# it, jobs are pushed right away.
|
|
10
|
+
#
|
|
11
|
+
# The jobs pushed inside the same transaction wait together and go in
|
|
12
|
+
# one round trip once it commits.
|
|
13
|
+
module TransactionalPush
|
|
14
|
+
# [the transaction, the deliveries waiting on it] for this thread.
|
|
15
|
+
WAITING = :robin_transactional_push
|
|
16
|
+
|
|
17
|
+
def self.call(_payloads, &deliver)
|
|
18
|
+
return false unless defined?(::ActiveRecord.after_all_transactions_commit)
|
|
19
|
+
|
|
20
|
+
transaction = current_transaction
|
|
21
|
+
waiting = Thread.current[WAITING]
|
|
22
|
+
if transaction && waiting&.first.equal?(transaction)
|
|
23
|
+
waiting.last << deliver
|
|
24
|
+
return true
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
deliveries = [deliver]
|
|
28
|
+
Thread.current[WAITING] = [transaction, deliveries] if transaction
|
|
29
|
+
ActiveRecord.after_all_transactions_commit do
|
|
30
|
+
Thread.current[WAITING] = nil if Thread.current[WAITING]&.last.equal?(deliveries)
|
|
31
|
+
Client.together { deliveries.each(&:call) }
|
|
32
|
+
end
|
|
33
|
+
true
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The innermost open transaction on the primary database, if any.
|
|
37
|
+
# Jobs pushed in it are delivered when the same transactions commit.
|
|
38
|
+
# One rolled back is never current again, so the deliveries left
|
|
39
|
+
# waiting on it are never added to (nor made).
|
|
40
|
+
def self.current_transaction
|
|
41
|
+
return unless defined?(ActiveRecord::Base.current_transaction)
|
|
42
|
+
|
|
43
|
+
transaction = ActiveRecord::Base.current_transaction
|
|
44
|
+
transaction if transaction.open?
|
|
45
|
+
end
|
|
46
|
+
private_class_method :current_transaction
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
data/lib/robin/rails.rb
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "robin"
|
|
4
|
+
require_relative "rails/app_loader"
|
|
5
|
+
require_relative "rails/error_reporter"
|
|
6
|
+
require_relative "rails/transactional_push"
|
|
7
|
+
require_relative "rails/current_attributes"
|
|
8
|
+
require_relative "rails/notifications"
|
|
9
|
+
require_relative "rails/railtie"
|
|
10
|
+
|
|
11
|
+
module Robin
|
|
12
|
+
# The server finds ActiveJob jobs' wrapper class by name, possibly before
|
|
13
|
+
# anything has loaded ActiveJob.
|
|
14
|
+
autoload :ActiveJob, "robin/active_job"
|
|
15
|
+
|
|
16
|
+
# Rails integration for Robin: a Railtie (see Robin::Rails::Railtie), the
|
|
17
|
+
# ActiveJob adapter, generators, Rails.error reporting and
|
|
18
|
+
# transaction-aware pushes. Robin itself works with any Ruby or Rack app:
|
|
19
|
+
# this is loaded with it in a Rails app (one that loaded Rails first, as
|
|
20
|
+
# Bundler.require does), or with `require "robin/rails"`.
|
|
21
|
+
module Rails
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Puts jobs held by crashed processes back on their queues.
|
|
5
|
+
#
|
|
6
|
+
# Each server registers its in-flight set (see Robin::Fetcher) in
|
|
7
|
+
# robin:inflights. A process whose heartbeat key has expired is dead;
|
|
8
|
+
# its in-flight jobs are moved back to the front of their queues. Jobs
|
|
9
|
+
# that keep getting orphaned (they crash their process every time) are
|
|
10
|
+
# moved to the dead set.
|
|
11
|
+
#
|
|
12
|
+
# A process that is shutting down writes a stopping:<identity> marker
|
|
13
|
+
# first; if it dies before it finishes, its jobs were interrupted by the
|
|
14
|
+
# shutdown, not crashing it, so recovering them doesn't count toward the
|
|
15
|
+
# poison-pill limit.
|
|
16
|
+
module Recovery
|
|
17
|
+
class << self
|
|
18
|
+
# Recovers every dead process's in-flight jobs, in three round trips
|
|
19
|
+
# however many there are (four when one is a poison pill). Returns
|
|
20
|
+
# the number of jobs recovered. +pool+ is where the Redis connection
|
|
21
|
+
# comes from (a server uses its housekeeping pool).
|
|
22
|
+
def run(config, pool: config.redis_pool)
|
|
23
|
+
pool.with do |conn|
|
|
24
|
+
# In order, so processes are logged (and recovered) in order.
|
|
25
|
+
sets = conn.call("SMEMBERS", config.key("inflights")).sort
|
|
26
|
+
prefix = "#{config.key("inflight")}:"
|
|
27
|
+
identities = sets.map { |set| set.delete_prefix(prefix) }
|
|
28
|
+
checks = conn.pipelined do |p|
|
|
29
|
+
identities.each do |identity|
|
|
30
|
+
p.call("EXISTS", config.process_key(identity))
|
|
31
|
+
p.call("EXISTS", config.key("stopping", identity))
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
dead = sets.zip(identities, checks.each_slice(2)).filter_map do |set, identity, (alive, stopping)|
|
|
35
|
+
[set, identity, stopping.zero?] if alive.zero?
|
|
36
|
+
end
|
|
37
|
+
recover(config, conn, dead).sum
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Moves all jobs in a process's in-flight keys (its in-flight set
|
|
42
|
+
# +key+) back to their queues, in one round trip (two when one is a
|
|
43
|
+
# poison pill), and unregisters the keys; returns how many. With
|
|
44
|
+
# +count+ false (a graceful shutdown) the jobs don't count toward
|
|
45
|
+
# the poison-pill limit.
|
|
46
|
+
def drain(config, conn, key, count:)
|
|
47
|
+
done = queue_recover(config, conn, key, count)
|
|
48
|
+
settle(config, conn, [[done, count]]).sum
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
# Recovers the jobs of +dead+ processes ([in-flight set, identity,
|
|
54
|
+
# count?] each) and forgets the processes; returns how many jobs each
|
|
55
|
+
# had.
|
|
56
|
+
def recover(config, conn, dead)
|
|
57
|
+
replies = conn.pipelined do |p|
|
|
58
|
+
dead.each do |set, identity, count|
|
|
59
|
+
queue_recover(config, p, set, count)
|
|
60
|
+
forget(config, p, set, identity)
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
counts = settle(config, conn, replies.each_slice(2).map(&:first).zip(dead.map(&:last)))
|
|
64
|
+
counts.zip(dead) do |recovered, (_set, identity, _count)|
|
|
65
|
+
next if recovered.zero?
|
|
66
|
+
|
|
67
|
+
config.logger.warn("recovered #{recovered} in-flight job(s) from dead process #{identity}")
|
|
68
|
+
config.statsd&.increment("jobs.recovered", recovered)
|
|
69
|
+
end
|
|
70
|
+
counts
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# mutant:disable -- Connection#evalsha splats ARGV, so passing the
|
|
74
|
+
# identity bare or wrapped in an Array is identical.
|
|
75
|
+
def forget(config, pipe, set, identity)
|
|
76
|
+
pipe.evalsha(Scripts::FORGET, [set, config.key("inflights"), config.key("stopping", identity), config.key("processes")], [identity])
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Runs RECOVER on +redis+ (a connection), or queues it (a pipeline).
|
|
80
|
+
def queue_recover(config, redis, key, count)
|
|
81
|
+
redis.evalsha(Scripts::RECOVER, [key, config.key("queues"), config.key("dead")],
|
|
82
|
+
["#{config.key("queue")}:", config.max_recoveries, Robin.epoch_seconds(Robin.now_ms), count, "#{config.key("orphan")}:"])
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Takes RECOVER's replies, each with whether its jobs count: marks
|
|
86
|
+
# the poison pills among them (in one round trip) and runs the
|
|
87
|
+
# recovery handlers. Returns how many jobs each reply had.
|
|
88
|
+
def settle(config, conn, results)
|
|
89
|
+
jobs = results.flat_map do |done, count|
|
|
90
|
+
# (RECOVER's reply is pairs, so each_slice(2) never yields a nil raw.)
|
|
91
|
+
done.each_slice(2).map { |status, raw| [status == "dead", raw, count] } # steep:ignore BlockBodyTypeMismatch
|
|
92
|
+
end #: Array[[bool, String, bool?]]
|
|
93
|
+
poison = jobs.select(&:first).map { |_dead, raw, _count| raw }
|
|
94
|
+
mark_poison(config, conn, poison) unless poison.empty?
|
|
95
|
+
jobs.each { |dead, raw, count| recovered_job(config, raw, dead) if count }
|
|
96
|
+
results.map { |done, _count| done.size / 2 }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Runs config.recovery_handlers for a job recovered after a crash.
|
|
100
|
+
def recovered_job(config, raw, poison)
|
|
101
|
+
return if config.recovery_handlers.empty? && config.event_handlers.empty?
|
|
102
|
+
|
|
103
|
+
payload = Robin.load_json(raw)
|
|
104
|
+
Events.emit(config, "job.recovered", job: payload, poison: poison)
|
|
105
|
+
config.recovery_handlers.each do |handler|
|
|
106
|
+
handler.call(payload, poison)
|
|
107
|
+
rescue => e
|
|
108
|
+
config.handle_exception(e, {context: "recovery handler", job: payload})
|
|
109
|
+
end
|
|
110
|
+
rescue JSON::ParserError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def mark_poison(config, conn, raws)
|
|
115
|
+
payloads = raws.map do |raw|
|
|
116
|
+
payload = Robin.load_json(raw).except("enqueued_at")
|
|
117
|
+
payload["error_class"] = "Robin::PoisonPill"
|
|
118
|
+
payload["error_message"] = "job was running when its process died #{config.max_recoveries + 1} times"
|
|
119
|
+
payload["failed_at"] ||= Robin.now_ms
|
|
120
|
+
payload
|
|
121
|
+
end
|
|
122
|
+
conn.multi do |tx|
|
|
123
|
+
raws.zip(payloads) do |raw, payload|
|
|
124
|
+
tx.call("ZREM", config.key("dead"), raw)
|
|
125
|
+
tx.call("ZADD", config.key("dead"), Robin.epoch_seconds(Robin.now_ms), Robin.dump_json(payload))
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
payloads.each do |payload|
|
|
129
|
+
config.logger.error("moved poison-pill job #{payload["class"]} jid=#{payload.fetch("jid")} to the dead set")
|
|
130
|
+
config.statsd&.increment("jobs.poison", tags: {class: payload["class"]})
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|