robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest/sha1"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Periodic (cron) jobs.
|
|
7
|
+
#
|
|
8
|
+
# Robin.configure_server do |config|
|
|
9
|
+
# config.periodic do |cron|
|
|
10
|
+
# cron.register "*/5 * * * *", "CleanupJob"
|
|
11
|
+
# cron.register "0 3 * * *", "NightlyReport", queue: "low", args: ["daily"],
|
|
12
|
+
# tz: "America/Chicago", description: "Email the nightly report"
|
|
13
|
+
# end
|
|
14
|
+
# end
|
|
15
|
+
#
|
|
16
|
+
# Every server knows the schedule, but only the cluster leader enqueues.
|
|
17
|
+
# Each tick is claimed with an atomic SET NX, so a tick runs exactly once
|
|
18
|
+
# even while leadership changes hands; after a failover, the most recent
|
|
19
|
+
# missed tick (only) is caught up.
|
|
20
|
+
#
|
|
21
|
+
# Storage:
|
|
22
|
+
# robin:periodic hash id => definition JSON (the registry)
|
|
23
|
+
# robin:periodic:armed hash id => ms when first registered
|
|
24
|
+
# robin:periodic:paused set paused ids
|
|
25
|
+
# robin:periodic:<id>:h list recent runs, newest first
|
|
26
|
+
# robin:periodic:<id>:t:<ms> claim key per tick
|
|
27
|
+
module Periodic
|
|
28
|
+
HISTORY = 100
|
|
29
|
+
MAX_SLEEP = 5
|
|
30
|
+
CLAIM_TTL = 7 * 86_400
|
|
31
|
+
|
|
32
|
+
# Claims the ticks due now, of entries neither paused nor registered
|
|
33
|
+
# after their tick, each with an atomic SET NX. Returns, for each, 1
|
|
34
|
+
# (won), 0 (lost) or -1 (skipped).
|
|
35
|
+
#
|
|
36
|
+
# KEYS: armed hash, paused set
|
|
37
|
+
# ARGV: claim key prefix ("<prefix>periodic:"), then per tick its
|
|
38
|
+
# entry's id, due time (ms) and claim TTL (seconds)
|
|
39
|
+
CLAIM = Redis::Script.new(<<~LUA)
|
|
40
|
+
local results = {}
|
|
41
|
+
for i = 2, #ARGV, 3 do
|
|
42
|
+
local id, due = ARGV[i], tonumber(ARGV[i + 1])
|
|
43
|
+
local result = -1
|
|
44
|
+
if redis.call('SISMEMBER', KEYS[2], id) == 0 and due >= tonumber(redis.call('HGET', KEYS[1], id) or '0') then
|
|
45
|
+
result = redis.call('SET', ARGV[1] .. id .. ':t:' .. ARGV[i + 1], '1', 'NX', 'EX', ARGV[i + 2]) and 1 or 0
|
|
46
|
+
end
|
|
47
|
+
results[#results + 1] = result
|
|
48
|
+
end
|
|
49
|
+
return results
|
|
50
|
+
LUA
|
|
51
|
+
|
|
52
|
+
# Raised by Periodic.verify! with every problem it found.
|
|
53
|
+
class Invalid < Robin::Error; end
|
|
54
|
+
|
|
55
|
+
# Collects registrations from config.periodic blocks.
|
|
56
|
+
class Registry
|
|
57
|
+
attr_reader :entries
|
|
58
|
+
|
|
59
|
+
# With a +problems+ array, a bad registration is noted there instead
|
|
60
|
+
# of raising, so every one can be reported.
|
|
61
|
+
def initialize(problems = nil)
|
|
62
|
+
@entries = {}
|
|
63
|
+
@problems = problems
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Returns the entry's id: a digest of its schedule, class, queue,
|
|
67
|
+
# arguments and zone, so it stays the same across deploys.
|
|
68
|
+
def register(cron, klass, queue: nil, args: [], tz: nil, description: nil, **options)
|
|
69
|
+
# @type ivar @problems: Array[String]
|
|
70
|
+
begin
|
|
71
|
+
Cron.new(cron, tz: tz) # validate early
|
|
72
|
+
rescue Cron::ParseError => e
|
|
73
|
+
raise unless @problems
|
|
74
|
+
|
|
75
|
+
@problems << "#{klass}: #{e}"
|
|
76
|
+
return
|
|
77
|
+
end
|
|
78
|
+
klass = klass.to_s
|
|
79
|
+
queue = queue&.to_s
|
|
80
|
+
id = Digest::SHA1.hexdigest(Robin.dump_json([cron, klass, queue, args, tz]))[0, 12] #: String
|
|
81
|
+
@entries[id] = {"cron" => cron, "class" => klass, "queue" => queue, "args" => args, "tz" => tz,
|
|
82
|
+
"description" => description, "options" => options.transform_keys(&:to_s)}
|
|
83
|
+
id
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
class << self
|
|
88
|
+
# Checks periodic registrations, for an app's test suite. Pass the
|
|
89
|
+
# block given to config.periodic, or a config to check all of its
|
|
90
|
+
# blocks (those registered by configure_server only run in servers):
|
|
91
|
+
#
|
|
92
|
+
# SCHEDULE = ->(cron) { cron.register "0 3 * * *", "NightlyReport" }
|
|
93
|
+
# config.periodic(&SCHEDULE) # the initializer
|
|
94
|
+
# Robin::Periodic.verify!(&SCHEDULE) # a test
|
|
95
|
+
#
|
|
96
|
+
# Raises Invalid listing every bad cron expression or time zone,
|
|
97
|
+
# class that doesn't exist or isn't a job, and arguments that JSON
|
|
98
|
+
# would change. Returns the entries otherwise.
|
|
99
|
+
def verify!(config = Robin.config, &block)
|
|
100
|
+
problems = [] #: Array[String]
|
|
101
|
+
registry = Registry.new(problems)
|
|
102
|
+
(block ? [block] : config.periodic_blocks).each { |b| b.call(registry) }
|
|
103
|
+
registry.entries.each_value { |entry| problems.concat(entry_problems(entry)) }
|
|
104
|
+
raise Invalid, "periodic jobs have problems:\n#{problems.map { |p| " #{p}" }.join("\n")}" unless problems.empty?
|
|
105
|
+
|
|
106
|
+
registry.entries
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Called by Launcher#run when config.periodic blocks exist.
|
|
110
|
+
def attach(launcher)
|
|
111
|
+
config = launcher.config
|
|
112
|
+
registry = Registry.new
|
|
113
|
+
config.periodic_blocks.each { |block| block.call(registry) }
|
|
114
|
+
entries = registry.entries
|
|
115
|
+
return if entries.empty?
|
|
116
|
+
|
|
117
|
+
write_registry(config, entries)
|
|
118
|
+
crons = entries.transform_values { |e| Cron.new(e.fetch("cron"), tz: e.fetch("tz")) }
|
|
119
|
+
Thread.new do
|
|
120
|
+
Thread.current.name = "robin.periodic"
|
|
121
|
+
run_loop(launcher, entries, crons)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Every periodic job, in two round trips. With +history+ (a count),
|
|
126
|
+
# each also has its job options and up to that many of its latest
|
|
127
|
+
# runs, newest first (see #history).
|
|
128
|
+
def list(config: Robin.config, history: nil)
|
|
129
|
+
defs, armed, paused = config.redis do |c|
|
|
130
|
+
c.pipelined do |p|
|
|
131
|
+
p.call("HGETALL", config.key("periodic"))
|
|
132
|
+
p.call("HGETALL", config.key("periodic", "armed"))
|
|
133
|
+
p.call("SMEMBERS", config.key("periodic", "paused"))
|
|
134
|
+
end
|
|
135
|
+
end #: Array[untyped]
|
|
136
|
+
defs = defs.each_slice(2).to_h
|
|
137
|
+
armed = armed.each_slice(2).to_h
|
|
138
|
+
ids = defs.keys.sort
|
|
139
|
+
runs = config.redis do |c|
|
|
140
|
+
c.pipelined do |p|
|
|
141
|
+
ids.each do |id|
|
|
142
|
+
key = config.key("periodic", id, "h")
|
|
143
|
+
history ? p.call("LRANGE", key, 0, history - 1) : p.call("LINDEX", key, 0)
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
ids.zip(runs).map do |id, reply|
|
|
148
|
+
d = Robin.load_json(defs.fetch(id))
|
|
149
|
+
# The latest runs, or the latest run (or nil).
|
|
150
|
+
recent = Array(reply).map { |raw| Robin.load_json(raw) }
|
|
151
|
+
last = recent.first
|
|
152
|
+
cron = parse(d)
|
|
153
|
+
next_time = cron&.next_time
|
|
154
|
+
{"id" => id, "cron" => d.fetch("cron"), "class" => d.fetch("class"), "queue" => d["queue"] || "default",
|
|
155
|
+
"args" => d.fetch("args"), "tz" => d["tz"], "description" => d["description"] || cron&.description || d.fetch("cron"),
|
|
156
|
+
"next_at" => next_time && Integer(next_time) * 1000, "last_at" => last&.fetch("at"), "last_jid" => last&.fetch("jid"),
|
|
157
|
+
"paused" => paused.include?(id), "armed_at" => armed[id]&.to_i} #: listing
|
|
158
|
+
.tap { |job| job.merge!({"options" => d.fetch("options"), "history" => recent}) if history }
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def history(id, config: Robin.config, limit: 25)
|
|
163
|
+
raw = config.redis { |c| c.call("LRANGE", config.key("periodic", id, "h"), 0, limit - 1) }
|
|
164
|
+
raw.map { |r| Robin.load_json(r) }
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Enqueues the job right away (outside its schedule). Returns the jid.
|
|
168
|
+
def enqueue_now(id, config: Robin.config)
|
|
169
|
+
raw = config.redis { |c| c.call("HGET", config.key("periodic"), id) }
|
|
170
|
+
raise ArgumentError, "no periodic job #{id.inspect}" unless raw
|
|
171
|
+
|
|
172
|
+
jid = push_job(config, id, Robin.load_json(raw))
|
|
173
|
+
record(config, [[id, Robin.now_ms, jid]], manual: true)
|
|
174
|
+
jid
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def pause!(id, config: Robin.config)
|
|
178
|
+
config.redis { |c| c.call("SADD", config.key("periodic", "paused"), id) }
|
|
179
|
+
true
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def unpause!(id, config: Robin.config)
|
|
183
|
+
config.redis { |c| c.call("SREM", config.key("periodic", "paused"), id) }
|
|
184
|
+
true
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Checks every entry and enqueues the ones whose latest tick hasn't
|
|
188
|
+
# been claimed yet. Safe to call from several processes at once.
|
|
189
|
+
#
|
|
190
|
+
# +claimed+ remembers, per entry, the latest tick this process tried
|
|
191
|
+
# to claim (won or lost): that tick's claim key outlives the time it
|
|
192
|
+
# stays the latest, so trying again could only fail. Passing the same
|
|
193
|
+
# hash on every call spares those round trips; the claim in Redis
|
|
194
|
+
# still decides who enqueues.
|
|
195
|
+
def tick(config, entries, crons, now = Time.now, claimed = {})
|
|
196
|
+
# The ticks due, minus those tried already.
|
|
197
|
+
dues = entries.each_key.filter_map do |id|
|
|
198
|
+
cron = crons.fetch(id)
|
|
199
|
+
due = cron.match?(now) ? Integer(now) : cron.previous_time(now)&.to_i
|
|
200
|
+
[id, due * 1000] if due
|
|
201
|
+
end - claimed.to_a #: Array[tick]
|
|
202
|
+
return [] if dues.empty?
|
|
203
|
+
|
|
204
|
+
won = claim(config, dues, crons, now, claimed)
|
|
205
|
+
pushed = Client.together { won.map { |id, due_ms| [id, due_ms, push_job(config, id, entries.fetch(id))] } } #: Array[[String, Integer, String?]]
|
|
206
|
+
record(config, pushed)
|
|
207
|
+
pushed.map(&:last)
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Writes +entries+ (Registry#entries) to Redis as the periodic jobs,
|
|
211
|
+
# arming the new ones from now: what #attach does as a server starts.
|
|
212
|
+
def write_registry(config, entries)
|
|
213
|
+
now = Robin.now_ms
|
|
214
|
+
config.redis do |c|
|
|
215
|
+
c.multi do |tx|
|
|
216
|
+
tx.call("DEL", config.key("periodic"))
|
|
217
|
+
tx.call("HSET", config.key("periodic"), *entries.flat_map { |id, e| [id, Robin.dump_json(e)] })
|
|
218
|
+
entries.each_key { |id| tx.call("HSETNX", config.key("periodic", "armed"), id, now) }
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
private
|
|
224
|
+
|
|
225
|
+
# Claims the ticks +dues+ ([id, due (ms)] each) of entries not paused
|
|
226
|
+
# or registered after them, in one round trip; returns the ones won.
|
|
227
|
+
# A claim outlives the tick's time as the entry's latest (until its
|
|
228
|
+
# next tick) by CLAIM_TTL, so a tick long ago is never claimed twice.
|
|
229
|
+
def claim(config, dues, crons, now, claimed)
|
|
230
|
+
args = dues.flat_map { |id, due_ms| [id, due_ms, CLAIM_TTL + Integer(crons.fetch(id).next_time(now) || now) - Integer(now)] }
|
|
231
|
+
results = config.redis do |c|
|
|
232
|
+
c.evalsha(CLAIM, [config.key("periodic", "armed"), config.key("periodic", "paused")], ["#{config.key("periodic")}:", *args])
|
|
233
|
+
end
|
|
234
|
+
dues.zip(results).filter_map do |(id, due_ms), result|
|
|
235
|
+
next if result.negative?
|
|
236
|
+
|
|
237
|
+
claimed[id] = due_ms
|
|
238
|
+
[id, due_ms] if result == 1
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def run_loop(launcher, entries, crons)
|
|
243
|
+
config = launcher.config
|
|
244
|
+
claimed = {} #: Hash[String, Integer]
|
|
245
|
+
until launcher.stopping?
|
|
246
|
+
begin
|
|
247
|
+
tick(config, entries, crons, Time.now, claimed) if launcher.leader.leader?
|
|
248
|
+
rescue => e
|
|
249
|
+
config.handle_exception(e, {context: "periodic jobs"})
|
|
250
|
+
end
|
|
251
|
+
sleep(seconds_until_next(crons, Time.now))
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Sleeps until the next tick is due, waking at least every MAX_SLEEP
|
|
256
|
+
# seconds to notice leadership changes and shutdown.
|
|
257
|
+
def seconds_until_next(crons, now)
|
|
258
|
+
soonest = crons.each_value.filter_map { |c| c.next_time(now) }.min
|
|
259
|
+
soonest ? (soonest - now).clamp(0.05, MAX_SLEEP) : MAX_SLEEP
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# A loaded class contributes its robin_options (queue, retry...);
|
|
263
|
+
# otherwise the name is pushed as is.
|
|
264
|
+
# mutant:disable -- `const_get` on Robin::Periodic instead of Object
|
|
265
|
+
# differs only for constants nested in Robin::Periodic, none of which
|
|
266
|
+
# are job classes.
|
|
267
|
+
def entry_problems(entry)
|
|
268
|
+
name = entry.fetch("class")
|
|
269
|
+
args = entry.fetch("args")
|
|
270
|
+
klass = begin
|
|
271
|
+
Object.const_get(name)
|
|
272
|
+
rescue NameError
|
|
273
|
+
nil
|
|
274
|
+
end
|
|
275
|
+
problems = [] #: Array[String]
|
|
276
|
+
if klass.nil?
|
|
277
|
+
problems << "#{name}: no such class"
|
|
278
|
+
elsif !(klass.instance_of?(Class) && (klass.method_defined?(:perform) || klass.respond_to?(:perform_later)))
|
|
279
|
+
problems << "#{name}: not a job class"
|
|
280
|
+
end
|
|
281
|
+
problems << "#{name}: args must be an Array" unless args.instance_of?(Array)
|
|
282
|
+
problems << "#{name}: args #{args.inspect} change in JSON" unless Robin.load_json(Robin.dump_json(args)) == args
|
|
283
|
+
problems
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
# mutant:disable -- `const_get` on Robin::Periodic instead of Object
|
|
287
|
+
# differs only for constants nested in Robin::Periodic, none of which
|
|
288
|
+
# are job classes.
|
|
289
|
+
def job_class(name)
|
|
290
|
+
Object.const_get(name)
|
|
291
|
+
rescue NameError
|
|
292
|
+
name
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def push_job(config, id, entry)
|
|
296
|
+
klass = job_class(entry.fetch("class"))
|
|
297
|
+
item = entry.fetch("options").merge({"class" => klass, "args" => entry.fetch("args"), "periodic" => id})
|
|
298
|
+
queue = entry.fetch("queue")
|
|
299
|
+
item["queue"] = queue if queue
|
|
300
|
+
Client.new(config: config).push(item)
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# Adds the pushes of ticks ([id, at (ms), jid] each) to their
|
|
304
|
+
# entries' histories, in one round trip.
|
|
305
|
+
def record(config, pushed, manual: false)
|
|
306
|
+
config.redis do |c|
|
|
307
|
+
c.pipelined do |p|
|
|
308
|
+
pushed.each do |id, at_ms, jid|
|
|
309
|
+
entry = {"at" => at_ms, "jid" => jid} #: Hash[String, untyped]
|
|
310
|
+
entry["manual"] = true if manual
|
|
311
|
+
key = config.key("periodic", id, "h")
|
|
312
|
+
p.call("LPUSH", key, Robin.dump_json(entry))
|
|
313
|
+
p.call("LTRIM", key, 0, HISTORY - 1)
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# nil for definitions this version can't parse.
|
|
320
|
+
def parse(entry)
|
|
321
|
+
Cron.new(entry.fetch("cron"), tz: entry["tz"])
|
|
322
|
+
rescue Cron::ParseError
|
|
323
|
+
nil
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
end
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "timeout"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# A worker thread: fetch a job, run it through middleware, record the
|
|
7
|
+
# outcome, repeat.
|
|
8
|
+
class Processor
|
|
9
|
+
# Seconds between fetch attempts while fetching fails (Redis is down,
|
|
10
|
+
# failing over, loading, out of memory...): doubles from 0.2s up to 5s.
|
|
11
|
+
MIN_BACKOFF = 0.1
|
|
12
|
+
MAX_BACKOFF = 5
|
|
13
|
+
|
|
14
|
+
attr_reader :capsule, :thread, :job
|
|
15
|
+
|
|
16
|
+
# The in-flight key of the last finished job, until its
|
|
17
|
+
# acknowledgement goes out with the next fetch (see Fetcher#fetch).
|
|
18
|
+
attr_reader :unacked
|
|
19
|
+
|
|
20
|
+
# [queue, raw payload, started at (ms), in-flight key] of the running
|
|
21
|
+
# job, or nil; kept up to date by the launcher.
|
|
22
|
+
attr_accessor :work
|
|
23
|
+
|
|
24
|
+
# When (monotonic seconds) it last finished a job, or nil, and how
|
|
25
|
+
# many it has finished; kept up to date by the launcher.
|
|
26
|
+
attr_accessor :finished_at, :finished
|
|
27
|
+
|
|
28
|
+
def initialize(manager, capsule, fetcher, slot)
|
|
29
|
+
@manager = manager
|
|
30
|
+
@capsule = capsule
|
|
31
|
+
@config = capsule.config
|
|
32
|
+
@fetcher = fetcher
|
|
33
|
+
@slot = slot
|
|
34
|
+
@retrier = Retrier.new(@config)
|
|
35
|
+
@job_logger = JobLogger.new(@config)
|
|
36
|
+
@wrapper = @config.job_wrapper
|
|
37
|
+
@failures = 0
|
|
38
|
+
@backoff = MIN_BACKOFF
|
|
39
|
+
@retired = false
|
|
40
|
+
@finished = 0
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Starts the worker thread, after +delay+ seconds.
|
|
44
|
+
def start(delay)
|
|
45
|
+
@thread = Thread.new do
|
|
46
|
+
Thread.current.name = "robin.#{capsule.name}.#{@slot}"
|
|
47
|
+
Thread.current.report_on_exception = false
|
|
48
|
+
Thread.current[:robin_capsule] = capsule
|
|
49
|
+
Events.emit(@config, "worker.started", capsule: capsule.name)
|
|
50
|
+
# Robin::Shutdown (sent by #kill) only interrupts the job itself or
|
|
51
|
+
# a sleep, never fetching, acknowledging or recording a failure.
|
|
52
|
+
Thread.handle_interrupt({Shutdown => :never}) { run(delay) }
|
|
53
|
+
rescue Shutdown
|
|
54
|
+
nil # delivered after the loop ended; nothing left to interrupt
|
|
55
|
+
end
|
|
56
|
+
self
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Stop after the current job.
|
|
60
|
+
def terminate
|
|
61
|
+
@done = true
|
|
62
|
+
@job&.interrupt_requested = true
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Stop after the current job, without interrupting it: the capsule
|
|
66
|
+
# needs fewer workers (see Manager#resize).
|
|
67
|
+
def retire
|
|
68
|
+
@retired = true
|
|
69
|
+
@done = true
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def retired? = @retired
|
|
73
|
+
|
|
74
|
+
def alive?
|
|
75
|
+
@thread&.alive?
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Hard stop: interrupt the running job.
|
|
79
|
+
def kill
|
|
80
|
+
@done = true
|
|
81
|
+
@thread&.raise(Shutdown) # a no-op for a thread that already finished
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Fetches the next job and runs it, on the calling thread, as the
|
|
85
|
+
# worker thread does each time round (its acknowledgement goes out
|
|
86
|
+
# with the next fetch, as there); what a benchmark measures. Returns
|
|
87
|
+
# whether there was one.
|
|
88
|
+
def work_once
|
|
89
|
+
unit = fetch
|
|
90
|
+
process(unit) if unit
|
|
91
|
+
!unit.nil?
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
private
|
|
95
|
+
|
|
96
|
+
def run(delay)
|
|
97
|
+
interruptible { sleep(delay) } if delay.positive?
|
|
98
|
+
until @done
|
|
99
|
+
unit = fetch
|
|
100
|
+
next unless unit
|
|
101
|
+
|
|
102
|
+
# Quieted while blocked waiting for work: hand the job back.
|
|
103
|
+
@done ? unit.requeue : process(unit)
|
|
104
|
+
end
|
|
105
|
+
rescue Shutdown
|
|
106
|
+
nil
|
|
107
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
108
|
+
died = e
|
|
109
|
+
ensure
|
|
110
|
+
acknowledge_unacked
|
|
111
|
+
release_fetched if @done
|
|
112
|
+
died ? @manager.processor_died(self, died) : @manager.processor_stopped(self)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# The next job, or nil. A failed fetch is reported (the first failure
|
|
116
|
+
# of an outage, then every 30th) and followed by a backoff sleep, so a
|
|
117
|
+
# Redis outage or error state never kills or spins the worker.
|
|
118
|
+
def fetch
|
|
119
|
+
ack = @unacked
|
|
120
|
+
@unacked = nil
|
|
121
|
+
unit = @fetcher.fetch(@slot, ack: ack)
|
|
122
|
+
if (started = @outage_started)
|
|
123
|
+
@config.logger.info(format("Redis is back after %.1fs", monotonic - started))
|
|
124
|
+
@outage_started = nil
|
|
125
|
+
@failures = 0
|
|
126
|
+
@backoff = MIN_BACKOFF
|
|
127
|
+
end
|
|
128
|
+
unit
|
|
129
|
+
rescue => e
|
|
130
|
+
@outage_started ||= monotonic
|
|
131
|
+
@failures += 1
|
|
132
|
+
@config.handle_exception(e, {context: "fetch"}) if @failures == 1 || (@failures % 30).zero?
|
|
133
|
+
@backoff = [@backoff * 2, MAX_BACKOFF].min
|
|
134
|
+
interruptible { sleep @backoff } unless @done
|
|
135
|
+
nil
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def process(unit)
|
|
139
|
+
payload = begin
|
|
140
|
+
Robin.remember_raw(Robin.load_json(unit.raw), unit.raw)
|
|
141
|
+
rescue JSON::ParserError => e
|
|
142
|
+
@config.handle_exception(e, {context: "invalid JSON job payload", queue: unit.queue, raw: unit.raw[0, 200]})
|
|
143
|
+
DeadSet.new(config: @config).kill(unit.raw, ack: unit)
|
|
144
|
+
return
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
if Cancellation.skip?(@config, payload["jid"], payload["bid"])
|
|
148
|
+
@config.logger.info { "skipping cancelled job #{payload["class"]} jid=#{payload.fetch("jid")}" }
|
|
149
|
+
Events.emit(@config, "job.skipped", job: payload, reason: "cancelled")
|
|
150
|
+
Batch.job_succeeded(@config, payload.fetch("bid"), payload.fetch("jid")) if payload["bid"]
|
|
151
|
+
complete(unit, payload)
|
|
152
|
+
return
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
if (paused = JobClass.paused_class(@config, payload)) && JobClass.park(@config, unit, paused)
|
|
156
|
+
@config.logger.info { "parked #{paused} jid=#{payload.fetch("jid")}: its class is paused" }
|
|
157
|
+
Events.emit(@config, "job.parked", job: payload, class: paused)
|
|
158
|
+
complete(unit, payload)
|
|
159
|
+
return
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
@manager.work_started(self, unit)
|
|
163
|
+
start = monotonic
|
|
164
|
+
ok = false
|
|
165
|
+
job_class = nil
|
|
166
|
+
begin
|
|
167
|
+
job_class = constantize(payload["class"])
|
|
168
|
+
execute(job_class, payload, unit)
|
|
169
|
+
ok = true
|
|
170
|
+
complete(unit, payload)
|
|
171
|
+
rescue Robin::Discard => e
|
|
172
|
+
@config.logger.info { "discarded #{payload.fetch("class")} jid=#{payload["jid"]}: #{Retrier.message_of(e)}" }
|
|
173
|
+
Events.emit(@config, "job.discarded", job: payload, error: e)
|
|
174
|
+
ok = true
|
|
175
|
+
complete(unit, payload)
|
|
176
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
177
|
+
raise if e.is_a?(SystemExit) || e.is_a?(NoMemoryError)
|
|
178
|
+
# Interrupted by a shutdown (directly, or a job turned the interrupt
|
|
179
|
+
# into its own error): leave the job in flight for the manager to
|
|
180
|
+
# requeue.
|
|
181
|
+
raise Shutdown if Robin.shutdown?(e)
|
|
182
|
+
|
|
183
|
+
handle_failure(unit, payload, job_class, e)
|
|
184
|
+
# Errors a job may raise (and its timeout) are recorded; anything
|
|
185
|
+
# more severe (e.g. a signal) is recorded and then takes the worker
|
|
186
|
+
# down with it.
|
|
187
|
+
raise unless e.is_a?(StandardError) || e.is_a?(ScriptError) || e.is_a?(Job::Timeout)
|
|
188
|
+
ensure
|
|
189
|
+
elapsed_ms = ((monotonic - start) * 1000).round(1)
|
|
190
|
+
@manager.work_finished(self, payload, elapsed_ms, ok)
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Acknowledges a finished job: with the next fetch, which saves a
|
|
195
|
+
# round trip. One that was recovered from a crashed process also has
|
|
196
|
+
# its poison-pill counter cleared, so it's acknowledged right away, in
|
|
197
|
+
# the same round trip as that.
|
|
198
|
+
def complete(unit, payload)
|
|
199
|
+
return @unacked = unit.key unless payload.key?("recovered")
|
|
200
|
+
|
|
201
|
+
@config.redis do |conn|
|
|
202
|
+
conn.pipelined do |p|
|
|
203
|
+
unit.acknowledge(p)
|
|
204
|
+
p.call("DEL", @config.key("orphan", payload.fetch("jid")))
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# A stopping (or dying) worker won't fetch again, so its last finished
|
|
210
|
+
# job is acknowledged on its own. If that fails, the job stays in
|
|
211
|
+
# flight and is put back later (run again: at least once).
|
|
212
|
+
def acknowledge_unacked
|
|
213
|
+
unacked = @unacked or return
|
|
214
|
+
|
|
215
|
+
@config.redis { |conn| @fetcher.acknowledge(conn, [unacked]) }
|
|
216
|
+
@unacked = nil
|
|
217
|
+
rescue => e
|
|
218
|
+
@config.handle_exception(e, {context: "acknowledging a finished job"})
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# A stopping worker hands back the jobs its capsule fetched for other
|
|
222
|
+
# workers (they're stopping too), so they run elsewhere instead of
|
|
223
|
+
# waiting for this process to stop. A job fetched afterwards by a
|
|
224
|
+
# worker still finishing up is handed back when that one stops.
|
|
225
|
+
def release_fetched
|
|
226
|
+
@fetcher.release
|
|
227
|
+
rescue => e
|
|
228
|
+
@config.handle_exception(e, {context: "handing back fetched jobs"})
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def execute(job_class, payload, unit)
|
|
232
|
+
# @type ivar @batch_failure: Batch::HeldFailure
|
|
233
|
+
job = job_class.new
|
|
234
|
+
job.jid = payload["jid"] if job.respond_to?(:jid=)
|
|
235
|
+
job.payload = payload if job.respond_to?(:payload=)
|
|
236
|
+
@job = job
|
|
237
|
+
run_job = lambda do
|
|
238
|
+
@job_logger.call(payload) do
|
|
239
|
+
@config.server_middleware.invoke(job, payload, unit.queue) do
|
|
240
|
+
perform(job, payload)
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
return wrapped(payload, run_job) unless payload.key?("bid")
|
|
245
|
+
|
|
246
|
+
# A batch job's failure is recorded with its retry (see
|
|
247
|
+
# #handle_failure), or else on the way out, as its middleware would.
|
|
248
|
+
@batch_failure = Batch::HeldFailure.new(payload)
|
|
249
|
+
Batch.holding_failure(@batch_failure) { wrapped(payload, run_job) }
|
|
250
|
+
record_batch_failure
|
|
251
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
252
|
+
record_batch_failure unless retried?(e)
|
|
253
|
+
raise
|
|
254
|
+
ensure
|
|
255
|
+
@job = nil
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def wrapped(payload, run_job) = (wrapper = @wrapper) ? wrapper.call(payload, &run_job) : run_job.call
|
|
259
|
+
|
|
260
|
+
# Does #process hand +exception+ to the retrier?
|
|
261
|
+
def retried?(exception)
|
|
262
|
+
!(exception.is_a?(Discard) || exception.is_a?(SystemExit) || exception.is_a?(NoMemoryError) || Robin.shutdown?(exception))
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# The batch failure the retrier didn't take: the job's error was
|
|
266
|
+
# swallowed, or turned into a discard or a shutdown, by an outer
|
|
267
|
+
# middleware.
|
|
268
|
+
def record_batch_failure
|
|
269
|
+
held = take_batch_failure
|
|
270
|
+
Batch.job_failed(@config, held.payload.fetch("bid"), held.payload.fetch("jid"), held.info) if held
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
# The held batch failure, if the job's batch middleware recorded one.
|
|
274
|
+
def take_batch_failure
|
|
275
|
+
held = @batch_failure
|
|
276
|
+
@batch_failure = nil
|
|
277
|
+
held if held&.info
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def perform(job, payload)
|
|
281
|
+
args = payload["args"]
|
|
282
|
+
interruptible do
|
|
283
|
+
if (limit = payload["timeout"])
|
|
284
|
+
Timeout.timeout(limit.to_f, Job::Timeout) { job.perform(*args) }
|
|
285
|
+
else
|
|
286
|
+
job.perform(*args)
|
|
287
|
+
end
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Lets a pending or later Robin::Shutdown interrupt the block.
|
|
292
|
+
def interruptible(&)
|
|
293
|
+
Thread.handle_interrupt({Shutdown => :immediate}, &)
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def handle_failure(unit, payload, job_class, exception)
|
|
297
|
+
batch_failure = take_batch_failure&.info
|
|
298
|
+
# In the job's logging context, like the job's own log lines.
|
|
299
|
+
Context.with(JobLogger.context_for(payload)) do
|
|
300
|
+
@config.handle_exception(exception, {context: "job raised", job: payload, queue: unit.queue})
|
|
301
|
+
end
|
|
302
|
+
@retrier.handle(unit, payload, job_class || Object, exception, batch_failure: batch_failure)
|
|
303
|
+
rescue => e
|
|
304
|
+
# Couldn't record the failure (e.g. Redis down). The job stays in
|
|
305
|
+
# flight and will be put back later.
|
|
306
|
+
@config.handle_exception(e, {context: "recording job failure", job: payload})
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def constantize(name)
|
|
310
|
+
Object.const_get(name)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def monotonic
|
|
314
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
end
|