robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Long-running jobs that survive deploys and restarts.
|
|
5
|
+
#
|
|
6
|
+
# Instead of +perform+, define how to enumerate the work and what to do
|
|
7
|
+
# with each item. Robin checkpoints the cursor as it goes; when the
|
|
8
|
+
# server shuts down (or the job is retried after an error) it resumes
|
|
9
|
+
# right after the last finished item instead of starting over.
|
|
10
|
+
#
|
|
11
|
+
# class BackfillAvatars
|
|
12
|
+
# include Robin::IterableJob
|
|
13
|
+
#
|
|
14
|
+
# def build_enumerator(account_id, cursor:)
|
|
15
|
+
# active_record_records_enumerator(User.where(account_id: account_id), cursor: cursor)
|
|
16
|
+
# end
|
|
17
|
+
#
|
|
18
|
+
# def each_iteration(user, account_id)
|
|
19
|
+
# user.regenerate_avatar!
|
|
20
|
+
# end
|
|
21
|
+
# end
|
|
22
|
+
#
|
|
23
|
+
# An enumerator yields +[item, cursor]+ pairs; the cursor is any
|
|
24
|
+
# JSON-native value identifying the item, and +build_enumerator+
|
|
25
|
+
# receives the cursor of the last finished item (nil on the first run).
|
|
26
|
+
#
|
|
27
|
+
# Optional hooks: on_start, on_resume, on_cancel, on_stop (interrupted
|
|
28
|
+
# or cancelled; after on_cancel), on_complete. Inside them and
|
|
29
|
+
# each_iteration, +cursor+ is the last finished item's cursor and
|
|
30
|
+
# +current_object+ the item being (or last) worked on.
|
|
31
|
+
#
|
|
32
|
+
# A run stops and requeues itself (resuming from its checkpoint, like
|
|
33
|
+
# after a deploy) once it has run for +max_iteration_runtime+ seconds:
|
|
34
|
+
# the robin_option of that name, or config.max_iteration_runtime. That
|
|
35
|
+
# keeps one huge job from holding a worker for hours.
|
|
36
|
+
#
|
|
37
|
+
# Every CHECKPOINT_SECONDS the cursor, the progress and a cancellation
|
|
38
|
+
# check share one Redis round trip, so a cancelled job stops within
|
|
39
|
+
# about that long (or after its current iteration, if that's longer).
|
|
40
|
+
module IterableJob
|
|
41
|
+
CHECKPOINT_SECONDS = 5
|
|
42
|
+
STATE_TTL = 30 * 86_400
|
|
43
|
+
|
|
44
|
+
def self.included(base)
|
|
45
|
+
base.include(Job)
|
|
46
|
+
base.extend(ClassMethods)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
module ClassMethods
|
|
50
|
+
# Iterable jobs run through #perform from Robin::IterableJob; a class
|
|
51
|
+
# that defines its own would silently skip the iteration machinery.
|
|
52
|
+
def method_added(name)
|
|
53
|
+
if name == :perform
|
|
54
|
+
raise ArgumentError, "#{self} includes Robin::IterableJob: define build_enumerator and each_iteration, not perform"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
super
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
attr_reader :cursor, :iterations, :current_object
|
|
62
|
+
|
|
63
|
+
def perform(*args)
|
|
64
|
+
config = self.class.robin_config
|
|
65
|
+
key = config.key("iter", jid)
|
|
66
|
+
state = config.redis { |c| c.call("HGETALL", key) }.each_slice(2).to_h
|
|
67
|
+
resumed = !state.empty?
|
|
68
|
+
@cursor = (Robin.load_json(state.fetch("cursor")).fetch("c") if resumed)
|
|
69
|
+
@iterations = state["iterations"].to_i
|
|
70
|
+
resumed ? on_resume : on_start
|
|
71
|
+
|
|
72
|
+
enum = build_enumerator(*args, cursor: @cursor)
|
|
73
|
+
if enum.nil?
|
|
74
|
+
finish(config, key, @iterations)
|
|
75
|
+
return
|
|
76
|
+
end
|
|
77
|
+
raise ArgumentError, "#{self.class}#build_enumerator must return an Enumerator" unless enum.respond_to?(:each)
|
|
78
|
+
|
|
79
|
+
# Enumerator sizes count the remaining items (nil or Infinity when
|
|
80
|
+
# unknown, and then progress is a count; see #progress_args).
|
|
81
|
+
remaining = (enum.size if enum.respond_to?(:size))
|
|
82
|
+
total = (remaining + @iterations if remaining.instance_of?(Integer))
|
|
83
|
+
max_runtime = payload["max_iteration_runtime"] || config.max_iteration_runtime
|
|
84
|
+
started = last_checkpoint = monotonic
|
|
85
|
+
warned = false
|
|
86
|
+
checkpointed = @iterations
|
|
87
|
+
previous = started
|
|
88
|
+
enum.each do |item, item_cursor|
|
|
89
|
+
@current_object = item
|
|
90
|
+
begin
|
|
91
|
+
around_iteration { each_iteration(item, *args) }
|
|
92
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
93
|
+
# Keep the progress made so far: a retry (or the next process)
|
|
94
|
+
# resumes after the last finished item instead of redoing it.
|
|
95
|
+
safe_checkpoint(config, key, total) if @iterations > checkpointed
|
|
96
|
+
raise
|
|
97
|
+
end
|
|
98
|
+
@cursor = item_cursor
|
|
99
|
+
@iterations += 1
|
|
100
|
+
|
|
101
|
+
now = monotonic
|
|
102
|
+
if !warned && now - previous > config.timeout
|
|
103
|
+
warn_slow(now - previous, config)
|
|
104
|
+
warned = true
|
|
105
|
+
end
|
|
106
|
+
previous = now
|
|
107
|
+
if now - last_checkpoint >= CHECKPOINT_SECONDS
|
|
108
|
+
last_checkpoint = now
|
|
109
|
+
checkpointed = @iterations
|
|
110
|
+
if checkpoint(config, key, total, check_cancel: true)
|
|
111
|
+
logger.info { "#{self.class} jid=#{jid} cancelled after #{@iterations} iterations" }
|
|
112
|
+
config.redis { |c| c.call("DEL", key) }
|
|
113
|
+
on_cancel
|
|
114
|
+
on_stop
|
|
115
|
+
return
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
reason = if interrupted?
|
|
119
|
+
"interrupted"
|
|
120
|
+
elsif max_runtime && now - started >= max_runtime
|
|
121
|
+
"reached its max_iteration_runtime (#{max_runtime}s)"
|
|
122
|
+
end
|
|
123
|
+
if reason
|
|
124
|
+
requeue(config, key, total, reason)
|
|
125
|
+
on_stop
|
|
126
|
+
return
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
finish(config, key, total)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Runs each iteration (the block): override to wrap them, e.g. in a
|
|
133
|
+
# transaction.
|
|
134
|
+
def around_iteration = yield
|
|
135
|
+
|
|
136
|
+
def on_start
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def on_resume
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def on_cancel
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def on_stop
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def on_complete
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# -- enumerator helpers ------------------------------------------------------
|
|
152
|
+
|
|
153
|
+
# Iterates an array; the cursor is the index.
|
|
154
|
+
def array_enumerator(array, cursor:)
|
|
155
|
+
start = cursor.nil? ? 0 : cursor + 1
|
|
156
|
+
Enumerator.new([array.size - start, 0].max) do |y|
|
|
157
|
+
(start...array.size).each { |i| y.yield array.fetch(i), i }
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Iterates records in primary-key order, loading +batch_size+ at a
|
|
162
|
+
# time; the cursor is the last finished record's id.
|
|
163
|
+
def active_record_records_enumerator(relation, cursor:, batch_size: 100)
|
|
164
|
+
Enumerator.new do |y|
|
|
165
|
+
active_record_batches(relation, cursor, batch_size) do |records|
|
|
166
|
+
records.each { |r| y.yield r, r.id }
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Like active_record_records_enumerator, but yields whole batches.
|
|
172
|
+
def active_record_batches_enumerator(relation, cursor:, batch_size: 100)
|
|
173
|
+
Enumerator.new do |y|
|
|
174
|
+
active_record_batches(relation, cursor, batch_size) { |records| y.yield records, records.last.id }
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Iterates a CSV (or any enumerable of rows); the cursor is the row
|
|
179
|
+
# index. Rows before the cursor are skipped, not re-processed.
|
|
180
|
+
def csv_enumerator(csv, cursor:)
|
|
181
|
+
Enumerator.new do |y|
|
|
182
|
+
csv.each_with_index { |row, i| y.yield row, i if cursor.nil? || i > cursor }
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
private
|
|
187
|
+
|
|
188
|
+
def active_record_batches(relation, cursor, batch_size)
|
|
189
|
+
pk = relation.primary_key
|
|
190
|
+
last = cursor
|
|
191
|
+
loop do
|
|
192
|
+
scope = relation.reorder({pk => :asc}).limit(batch_size)
|
|
193
|
+
scope = scope.where(relation.arel_table[pk].gt(last)) unless last.nil?
|
|
194
|
+
records = scope.to_a
|
|
195
|
+
break if records.empty?
|
|
196
|
+
|
|
197
|
+
yield records
|
|
198
|
+
last = records.last.id
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Checkpoints while the job is already failing; a Redis error here is
|
|
203
|
+
# reported but mustn't replace the job's own exception.
|
|
204
|
+
def safe_checkpoint(config, key, total)
|
|
205
|
+
checkpoint(config, key, total)
|
|
206
|
+
rescue => e
|
|
207
|
+
config.handle_exception(e, {context: "iterable job checkpoint", job: payload})
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# One iteration outlasted the shutdown timeout: a deploy could interrupt
|
|
211
|
+
# it midway, and it would run again.
|
|
212
|
+
def warn_slow(seconds, config)
|
|
213
|
+
logger.warn("#{self.class} jid=#{jid}: one iteration took #{seconds.round(1)}s, longer than the " \
|
|
214
|
+
"#{config.timeout}s shutdown timeout; it may be interrupted midway and repeated")
|
|
215
|
+
Events.emit(config, "iteration.slow", job: payload, seconds: seconds)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# Saves the cursor in one MULTI, followed by whatever the block queues
|
|
219
|
+
# (so a requeued job always finds its checkpoint). With +check_cancel+,
|
|
220
|
+
# returns whether the job (or its batch) was cancelled. When the job's
|
|
221
|
+
# Redis is also the one progress! and cancelled? use (the usual case),
|
|
222
|
+
# the progress and the cancellation check ride in the same MULTI.
|
|
223
|
+
def checkpoint(config, key, total, check_cancel: false)
|
|
224
|
+
shared = config.equal?(Robin.config)
|
|
225
|
+
replies = config.redis do |c|
|
|
226
|
+
c.multi do |tx|
|
|
227
|
+
tx.call("HSET", key, "cursor", Robin.dump_json({"c" => @cursor}), "iterations", @iterations, "at", Robin.now_ms)
|
|
228
|
+
tx.call("EXPIRE", key, STATE_TTL)
|
|
229
|
+
tx.call(*progress_command(config, total)) if shared
|
|
230
|
+
yield tx if block_given?
|
|
231
|
+
# Robin::Cancellation.cancelled?, inline.
|
|
232
|
+
tx.call("ZMSCORE", config.key("cancelled"), *[jid, bid && "batch:#{bid}"].compact) if shared && check_cancel
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
report_progress(total) unless shared
|
|
236
|
+
check_cancel && (shared ? replies.last.any? : cancelled?)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# How far along the job is, for Progress ([percent, message, meta]): a
|
|
240
|
+
# percentage when the enumerator's size gave the total, or else no
|
|
241
|
+
# percentage and the count of finished iterations ("iterations"),
|
|
242
|
+
# which the Busy page shows instead.
|
|
243
|
+
def progress_args(total)
|
|
244
|
+
total ? [percent(total), "#{@iterations}/#{total}", {}] : [nil, nil, {"iterations" => @iterations}]
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# An empty enumerator is all done.
|
|
248
|
+
# mutant:disable -- Progress clamps percentages, so any above 100 is 100.
|
|
249
|
+
def percent(total) = total.zero? ? 100 : @iterations * 100.0 / total
|
|
250
|
+
|
|
251
|
+
def progress_command(config, total)
|
|
252
|
+
pct, message, meta = progress_args(total)
|
|
253
|
+
Progress.set_command(config, jid, Progress.entry(pct, message, meta))
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def report_progress(total)
|
|
257
|
+
pct, message, meta = progress_args(total)
|
|
258
|
+
Progress.report(jid, pct, message, meta)
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# Puts the job back on its queue (same jid) so any process can resume
|
|
262
|
+
# it from the checkpoint. Bypasses client middleware: this isn't a new
|
|
263
|
+
# job, so uniqueness and batch registration must not apply again.
|
|
264
|
+
def requeue(config, key, total, reason)
|
|
265
|
+
# As stored, so decrypted (or mutated) arguments never are.
|
|
266
|
+
item = Robin.pristine(payload).dup
|
|
267
|
+
%w[at error_class error_message error_backtrace failed_at retried_at].each { |k| item.delete(k) }
|
|
268
|
+
item["enqueued_at"] = Robin.now_ms
|
|
269
|
+
queue = item["queue"] || "default"
|
|
270
|
+
checkpoint(config, key, total) do |tx|
|
|
271
|
+
tx.call("SADD", config.key("queues"), queue)
|
|
272
|
+
tx.call("RPUSH", config.key("queue", queue), Robin.dump_json(item))
|
|
273
|
+
end
|
|
274
|
+
payload["iteration_interrupted"] = true
|
|
275
|
+
logger.info { "#{self.class} jid=#{jid} #{reason} after #{@iterations} iterations; requeued" }
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# Drops the saved state (and reports 100%, or the count) before
|
|
279
|
+
# on_complete runs.
|
|
280
|
+
def finish(config, key, total)
|
|
281
|
+
shared = config.equal?(Robin.config)
|
|
282
|
+
config.redis do |c|
|
|
283
|
+
c.pipelined do |p|
|
|
284
|
+
p.call("DEL", key)
|
|
285
|
+
p.call(*progress_command(config, total)) if shared
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
report_progress(total) unless shared
|
|
289
|
+
on_complete
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
293
|
+
end
|
|
294
|
+
end
|
data/lib/robin/job.rb
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Include in a class to make it a Robin job.
|
|
5
|
+
#
|
|
6
|
+
# class Thumbnail
|
|
7
|
+
# include Robin::Job
|
|
8
|
+
# robin_options queue: "media", retry: 10, expires_in: 3600
|
|
9
|
+
#
|
|
10
|
+
# def perform(image_id, size)
|
|
11
|
+
# ...
|
|
12
|
+
# end
|
|
13
|
+
# end
|
|
14
|
+
#
|
|
15
|
+
# Supported options (all optional):
|
|
16
|
+
#
|
|
17
|
+
# queue:: queue name (default "default")
|
|
18
|
+
# retry:: true, false, or a maximum number of retries
|
|
19
|
+
# retry_for:: seconds after the first failure to keep retrying; without
|
|
20
|
+
# an explicit retry count, it replaces config.max_retries
|
|
21
|
+
# retry_queue:: queue the job's retries run on (default: its own)
|
|
22
|
+
# dead:: false to discard instead of moving to the dead set
|
|
23
|
+
# backtrace:: true or a number of lines to keep on failures
|
|
24
|
+
# tags:: array of strings shown and searchable in the web UI
|
|
25
|
+
# expires_in:: seconds after enqueue that the job becomes worthless
|
|
26
|
+
# unique_for:: seconds to suppress duplicate pushes (same class+args)
|
|
27
|
+
# unique_until:: :start or :success (default) - when the lock is released
|
|
28
|
+
# debounce:: seconds; later pushes with the same args replace earlier ones
|
|
29
|
+
# encrypt:: true to encrypt arguments at rest
|
|
30
|
+
# timeout:: seconds the job may run before a Robin::Job::Timeout is raised
|
|
31
|
+
# log_level:: level of the job's start/done/fail lines (default :info)
|
|
32
|
+
# logging:: false to leave those lines out
|
|
33
|
+
# profile:: true or a label to profile the run (see Robin::Profiler)
|
|
34
|
+
module Job
|
|
35
|
+
# Raised inside a job that runs past its +timeout+. Not a
|
|
36
|
+
# StandardError, so a job's own `rescue => e` can't swallow it and run
|
|
37
|
+
# on. Like any Timeout.timeout, it can strike anywhere: inside an
|
|
38
|
+
# ensure block, or halfway through talking to a database, leaving
|
|
39
|
+
# connections or state half-updated. Prefer timeouts on the slow
|
|
40
|
+
# operations themselves, and treat this as a last resort.
|
|
41
|
+
class Timeout < Exception; end # rubocop:disable Lint/InheritException
|
|
42
|
+
|
|
43
|
+
@inclusion_checks = []
|
|
44
|
+
|
|
45
|
+
class << self
|
|
46
|
+
# Callables run with each class about to include Robin::Job; one
|
|
47
|
+
# raises to reject the class, which is then left untouched.
|
|
48
|
+
# Integrations use it: the Rails integration refuses ActiveJob classes.
|
|
49
|
+
attr_reader :inclusion_checks
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def self.append_features(base)
|
|
53
|
+
inclusion_checks.each { |check| check.call(base) }
|
|
54
|
+
super
|
|
55
|
+
base.extend(ClassMethods)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Option hashes use string keys, and symbol values become strings: the
|
|
59
|
+
# same shape the payload has after its JSON round trip.
|
|
60
|
+
def self.stringify(hash)
|
|
61
|
+
hash.map { |k, v| [k.to_s, v.instance_of?(Symbol) ? v.to_s : v] }.to_h
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
attr_accessor :jid, :payload, :interrupt_requested
|
|
65
|
+
|
|
66
|
+
def logger
|
|
67
|
+
Robin.logger
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def queue
|
|
71
|
+
payload && payload["queue"]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def batch
|
|
75
|
+
bid && Batch.new(bid)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def bid
|
|
79
|
+
payload && payload["bid"]
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# How many times this job has failed before.
|
|
83
|
+
def failures
|
|
84
|
+
count = payload && payload["retry_count"]
|
|
85
|
+
count ? Integer(count) + 1 : 0
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# The capsule running the job (its name, queues and concurrency), or
|
|
89
|
+
# nil when it isn't run by a server's worker (perform_inline, tests).
|
|
90
|
+
def capsule = Capsule.current
|
|
91
|
+
|
|
92
|
+
# True once someone called Robin.cancel!(jid), or cancelled the job's
|
|
93
|
+
# batch. Long jobs should check this periodically and return early.
|
|
94
|
+
def cancelled?
|
|
95
|
+
Cancellation.cancelled?(jid, bid: bid)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Report progress (0-100, or nil when the job can't tell) with an
|
|
99
|
+
# optional message; shown live in the web UI's Busy page.
|
|
100
|
+
def progress!(percent, message = nil, **meta)
|
|
101
|
+
Progress.report(jid, percent, message, meta)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# True while the server is shutting down; iterable jobs use this to
|
|
105
|
+
# checkpoint and stop between iterations.
|
|
106
|
+
def interrupted?
|
|
107
|
+
!!@interrupt_requested
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
module ClassMethods
|
|
111
|
+
RESERVED = %w[class args jid created_at enqueued_at at bid].freeze
|
|
112
|
+
|
|
113
|
+
def robin_options(opts = nil)
|
|
114
|
+
# @type ivar @robin_options: job_hash
|
|
115
|
+
if opts
|
|
116
|
+
@robin_options = robin_options.merge(Job.stringify(opts))
|
|
117
|
+
validate_options!(@robin_options)
|
|
118
|
+
end
|
|
119
|
+
@robin_options || (superclass.respond_to?(:robin_options) ? superclass.robin_options : DEFAULT_OPTIONS)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# Custom retry delay: the block receives (count, exception, payload)
|
|
123
|
+
# and returns seconds, :discard, :kill, or nil for the default.
|
|
124
|
+
def robin_retry_in(&block)
|
|
125
|
+
@robin_retry_in = block
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def robin_retry_in_block
|
|
129
|
+
@robin_retry_in || (superclass.robin_retry_in_block if superclass.respond_to?(:robin_retry_in_block))
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Called when a job runs out of retries, with (payload, exception).
|
|
133
|
+
def robin_retries_exhausted(&block)
|
|
134
|
+
@robin_retries_exhausted = block
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def robin_retries_exhausted_block
|
|
138
|
+
@robin_retries_exhausted || (superclass.robin_retries_exhausted_block if superclass.respond_to?(:robin_retries_exhausted_block))
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def set(options)
|
|
142
|
+
Setter.new(self, options)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def perform_async(*)
|
|
146
|
+
Setter.new(self, {}).perform_async(*)
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def perform_in(interval, *)
|
|
150
|
+
Setter.new(self, {}).perform_in(interval, *)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def perform_at(time, *)
|
|
154
|
+
Setter.new(self, {}).perform_at(time, *)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def perform_bulk(args_list, **)
|
|
158
|
+
Setter.new(self, {}).perform_bulk(args_list, **)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Runs the job immediately in the current thread, through the same
|
|
162
|
+
# argument round-trip (JSON) a real job would take.
|
|
163
|
+
def perform_inline(*)
|
|
164
|
+
Setter.new(self, {}).perform_inline(*)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def client_push(item)
|
|
168
|
+
Client.new(config: robin_config).push(item)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def robin_config
|
|
172
|
+
Robin.config
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
private
|
|
176
|
+
|
|
177
|
+
def validate_options!(opts)
|
|
178
|
+
bad = opts.keys & RESERVED
|
|
179
|
+
raise ArgumentError, "reserved robin_options: #{bad.join(", ")}" unless bad.empty?
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
DEFAULT_OPTIONS = {"queue" => "default", "retry" => true}.freeze
|
|
184
|
+
BULK_BATCH_SIZE = 1_000
|
|
185
|
+
|
|
186
|
+
# Holds per-push overrides: MyJob.set(queue: "critical", wait: 30).perform_async
|
|
187
|
+
#
|
|
188
|
+
# +pool:+ pushes to another Redis (see Client.via):
|
|
189
|
+
# MyJob.set(pool: SHARDS.sample).perform_async(1)
|
|
190
|
+
class Setter
|
|
191
|
+
def initialize(klass, options)
|
|
192
|
+
@klass = klass
|
|
193
|
+
@options = {}
|
|
194
|
+
set(options)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def set(options)
|
|
198
|
+
options = Job.stringify(options)
|
|
199
|
+
@pool = options.delete("pool") if options.key?("pool")
|
|
200
|
+
@options.merge!(options)
|
|
201
|
+
self
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def perform_async(*args)
|
|
205
|
+
if (wait = @options.delete("wait"))
|
|
206
|
+
return perform_in(wait, *args)
|
|
207
|
+
end
|
|
208
|
+
if (wait_until = @options.delete("wait_until"))
|
|
209
|
+
return perform_at(wait_until, *args)
|
|
210
|
+
end
|
|
211
|
+
if @options.delete("sync")
|
|
212
|
+
return perform_inline(*args)
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
via_pool { @klass.client_push(item(args)) }
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# The client turns +interval+ (seconds, or a time) into a timestamp;
|
|
219
|
+
# times in the past enqueue the job right away.
|
|
220
|
+
def perform_in(interval, *args)
|
|
221
|
+
via_pool { @klass.client_push(item(args).merge({"at" => interval})) }
|
|
222
|
+
end
|
|
223
|
+
alias_method :perform_at, :perform_in
|
|
224
|
+
|
|
225
|
+
# +spread_interval+ (seconds) spreads the jobs evenly over that long,
|
|
226
|
+
# across every slice of +batch_size+.
|
|
227
|
+
def perform_bulk(args_list, batch_size: BULK_BATCH_SIZE, spread_interval: nil)
|
|
228
|
+
client = Client.new(config: @klass.robin_config, pool: @pool)
|
|
229
|
+
ats = Client.spread(args_list.size, spread_interval) if spread_interval
|
|
230
|
+
args_list.each_slice(batch_size).with_index.flat_map do |slice, n|
|
|
231
|
+
bulk = item(nil).merge({"args" => slice})
|
|
232
|
+
bulk["at"] = ats[n * batch_size, batch_size] if ats
|
|
233
|
+
client.push_bulk(bulk)
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def perform_inline(*args)
|
|
238
|
+
payload = Client.new(config: @klass.robin_config).normalize(item(args))
|
|
239
|
+
payload = Robin.load_json(Robin.dump_json(payload))
|
|
240
|
+
job = @klass.new
|
|
241
|
+
job.jid = payload.fetch("jid")
|
|
242
|
+
job.payload = payload
|
|
243
|
+
job.perform(*payload.fetch("args"))
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
private
|
|
247
|
+
|
|
248
|
+
def via_pool(&)
|
|
249
|
+
@pool ? Client.via(@pool, &) : yield
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
# The client merges in the class's robin_options underneath these.
|
|
253
|
+
def item(args)
|
|
254
|
+
@options.merge({"class" => @klass, "args" => args})
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
end
|