robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
data/lib/robin/unique.rb
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest/sha2"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Unique jobs: suppress pushes of a job identical (same class, queue and
|
|
7
|
+
# arguments) to one that's still pending.
|
|
8
|
+
#
|
|
9
|
+
# class SyncAccount
|
|
10
|
+
# include Robin::Job
|
|
11
|
+
# robin_options unique_for: 600 # seconds
|
|
12
|
+
# robin_options unique_until: :start # default :success
|
|
13
|
+
#
|
|
14
|
+
# # Optional: decide which arguments make a job unique.
|
|
15
|
+
# def self.robin_unique_args(args) = args.first(1)
|
|
16
|
+
# end
|
|
17
|
+
#
|
|
18
|
+
# SyncAccount.perform_async(1) # => "jid..."
|
|
19
|
+
# SyncAccount.perform_async(1) # => nil (duplicate)
|
|
20
|
+
#
|
|
21
|
+
# The lock lives until the job starts (:start) or succeeds (:success), or
|
|
22
|
+
# until unique_for seconds pass — whichever is first. For scheduled jobs
|
|
23
|
+
# the lock also covers the wait until the scheduled time. Failed jobs
|
|
24
|
+
# keep the lock while they wait for a retry; it is released when they
|
|
25
|
+
# die or are discarded.
|
|
26
|
+
module Unique
|
|
27
|
+
# Locks taken ahead for a bulk push (see Unique.claiming).
|
|
28
|
+
CLAIMS = :robin_unique_claims
|
|
29
|
+
|
|
30
|
+
class << self
|
|
31
|
+
def digest(klass_name, queue, args)
|
|
32
|
+
klass = constant(klass_name)
|
|
33
|
+
args = klass.robin_unique_args(args) if klass.respond_to?(:robin_unique_args)
|
|
34
|
+
Digest::SHA256.hexdigest(Robin.dump_json([klass_name, queue, canonical(args)]))
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Hashes become key-sorted pairs (string keys), so argument hashes
|
|
38
|
+
# that differ only in key order or key type have the same digest.
|
|
39
|
+
def canonical(obj)
|
|
40
|
+
case obj
|
|
41
|
+
when Hash then obj.map { |k, v| [k.to_s, canonical(v)] }.sort_by(&:first)
|
|
42
|
+
when Array then obj.map { |v| canonical(v) }
|
|
43
|
+
else obj
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def lock_key(config, digest) = config.key("unique", digest)
|
|
48
|
+
|
|
49
|
+
# Does a job take a lock when it's pushed? Not without a positive
|
|
50
|
+
# unique_for, nor once it has its lock (e.g. a retry).
|
|
51
|
+
def lockable?(payload)
|
|
52
|
+
ttl = payload["unique_for"]
|
|
53
|
+
ttl&.positive? && !payload.key?("unique_digest")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Runs the block (a bulk push of +payloads+, which share their
|
|
57
|
+
# options) with their unique locks taken beforehand, in one round
|
|
58
|
+
# trip, when +chain+ (client middleware) starts with
|
|
59
|
+
# ClientMiddleware: nothing before it can change a job or stop its
|
|
60
|
+
# push then. Only a lock's first job in +payloads+ claims it ahead:
|
|
61
|
+
# another with the same lock tries when its turn comes, as it would
|
|
62
|
+
# have, after the first one's push went through or was stopped.
|
|
63
|
+
# Locks taken for jobs the block never got to (it raised) are given
|
|
64
|
+
# back.
|
|
65
|
+
def claiming(chain, payloads)
|
|
66
|
+
entry = chain.first
|
|
67
|
+
first = payloads.first
|
|
68
|
+
return yield unless first && entry&.klass.equal?(ClientMiddleware) && lockable?(first)
|
|
69
|
+
|
|
70
|
+
# (entry&.klass.equal? above means entry is there, and that it builds
|
|
71
|
+
# a ClientMiddleware; Steep can't tell.)
|
|
72
|
+
middleware = entry.build # steep:ignore NoMethod
|
|
73
|
+
claims = middleware.claim_all(payloads)
|
|
74
|
+
previous = Thread.current[CLAIMS]
|
|
75
|
+
Thread.current[CLAIMS] = claims
|
|
76
|
+
begin
|
|
77
|
+
yield
|
|
78
|
+
ensure
|
|
79
|
+
Thread.current[CLAIMS] = previous
|
|
80
|
+
left = claims.select { |_payload, (_digest, _key, won)| won }
|
|
81
|
+
unless left.empty?
|
|
82
|
+
middleware.redis { |c| c.pipelined { |p| left.each { |payload, (_, key)| compare_and_delete(p, key, payload.fetch("jid")) } } }
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Is a lock currently held for these arguments?
|
|
88
|
+
def locked?(klass_name, args, queue: "default", config: Robin.config)
|
|
89
|
+
config.redis { |c| c.call("EXISTS", lock_key(config, digest(klass_name, queue, args))) } == 1
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def release(config, payload) = payload["unique_digest"] && config.redis { |c| eval_release(c, config, payload) }
|
|
93
|
+
|
|
94
|
+
# Releases +payload+'s lock, if it has one and still holds it, on
|
|
95
|
+
# +redis+: a connection (returns the reply), or a pipeline or
|
|
96
|
+
# transaction to queue it in.
|
|
97
|
+
def eval_release(redis, config, payload)
|
|
98
|
+
digest = payload["unique_digest"]
|
|
99
|
+
compare_and_delete(redis, lock_key(config, digest), payload.fetch("jid")) if digest
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
private
|
|
103
|
+
|
|
104
|
+
# mutant:disable -- Connection#evalsha splats ARGV, so passing the
|
|
105
|
+
# owner bare or wrapped in an Array is identical.
|
|
106
|
+
def compare_and_delete(redis, key, owner) = redis.evalsha(Scripts::RELEASE, [key], [owner])
|
|
107
|
+
|
|
108
|
+
# mutant:disable -- `const_get` on Robin::Unique instead of Object
|
|
109
|
+
# differs only for constants nested in Robin::Unique, none of which
|
|
110
|
+
# are job classes.
|
|
111
|
+
def constant(name)
|
|
112
|
+
Object.const_get(name)
|
|
113
|
+
rescue NameError
|
|
114
|
+
nil
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
class ClientMiddleware
|
|
119
|
+
include Middleware::ConfigAware
|
|
120
|
+
include Middleware::Stateless
|
|
121
|
+
|
|
122
|
+
def self.applies?(_job_class, payload, _queue, _redis) = Unique.lockable?(payload)
|
|
123
|
+
|
|
124
|
+
def call(job_class, payload, queue, _redis)
|
|
125
|
+
return yield unless Unique.lockable?(payload)
|
|
126
|
+
|
|
127
|
+
digest, key, won = Thread.current[CLAIMS]&.delete(payload) || claim(job_class, payload, queue)
|
|
128
|
+
unless won
|
|
129
|
+
logger.debug { "suppressed duplicate #{job_class} #{payload.fetch("args")}; locked by #{redis { |c| c.call("GET", key) }}" }
|
|
130
|
+
return
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
payload["unique_digest"] = digest
|
|
134
|
+
result = yield
|
|
135
|
+
Unique.release(config, payload) unless result # a later middleware stopped the push
|
|
136
|
+
result
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Takes the locks of +payloads+ (see Unique.claiming) in one round
|
|
140
|
+
# trip; returns {payload => [digest, lock key, won?]}, by identity.
|
|
141
|
+
def claim_all(payloads)
|
|
142
|
+
locks = {} #: Hash[String, [job_hash, [String, String, Integer]]]
|
|
143
|
+
payloads.each do |payload|
|
|
144
|
+
next unless Unique.lockable?(payload)
|
|
145
|
+
|
|
146
|
+
lock = lock_for(payload.fetch("class"), payload, payload.fetch("queue"))
|
|
147
|
+
locks[lock[1]] ||= [payload, lock]
|
|
148
|
+
end
|
|
149
|
+
replies = redis { |c| c.pipelined { |p| locks.each_value { |payload, (_, key, expires_at)| p.call(*set_command(payload, key, expires_at)) } } }
|
|
150
|
+
locks.values.zip(replies).to_h { |(payload, (digest, key, _)), won| [payload, [digest, key, won]] }.compare_by_identity #: Hash[job_hash, claim]
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
private
|
|
154
|
+
|
|
155
|
+
# Takes the lock for one job: [digest, lock key, won?].
|
|
156
|
+
def claim(job_class, payload, queue)
|
|
157
|
+
digest, key, expires_at = lock_for(job_class, payload, queue)
|
|
158
|
+
[digest, key, redis { |c| c.call(*set_command(payload, key, expires_at)) }]
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# [digest, lock key, expiry (epoch ms)] for a job.
|
|
162
|
+
def lock_for(job_class, payload, queue)
|
|
163
|
+
digest = Unique.digest(job_class, queue, payload.fetch("args"))
|
|
164
|
+
# Scheduled jobs stay locked until unique_for after their run time.
|
|
165
|
+
expires_at = [payload["at"]&.then { |at| Robin.epoch_ms(at) }, Robin.now_ms].compact.max + Integer(payload.fetch("unique_for") * 1000)
|
|
166
|
+
[digest, Unique.lock_key(config, digest), expires_at]
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def set_command(payload, key, expires_at) = ["SET", key, payload.fetch("jid"), "NX", "PXAT", expires_at]
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
class ServerMiddleware
|
|
173
|
+
include Middleware::ConfigAware
|
|
174
|
+
include Middleware::Stateless
|
|
175
|
+
|
|
176
|
+
def self.applies?(_job, payload, _queue) = payload.key?("unique_digest")
|
|
177
|
+
|
|
178
|
+
def call(_job, payload, _queue)
|
|
179
|
+
# With unique_until: :start the lock goes before the job runs, and
|
|
180
|
+
# nothing takes it again for this jid, so that's the only release.
|
|
181
|
+
if payload["unique_until"].to_s == "start"
|
|
182
|
+
Unique.release(config, payload)
|
|
183
|
+
return yield
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
begin
|
|
187
|
+
result = yield
|
|
188
|
+
rescue Robin::Discard
|
|
189
|
+
Unique.release(config, payload)
|
|
190
|
+
raise
|
|
191
|
+
end
|
|
192
|
+
# An interrupted iterable job was requeued and is still pending.
|
|
193
|
+
Unique.release(config, payload) unless payload["iteration_interrupted"]
|
|
194
|
+
result
|
|
195
|
+
end
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
end
|
data/lib/robin/warmup.rb
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Warms up the JIT before a server starts (see Config#jit_warmup): runs
|
|
5
|
+
# the code that pushes, fetches and runs jobs many times over, so it's
|
|
6
|
+
# compiled before real jobs come. A cold JRuby process pushes and runs
|
|
7
|
+
# jobs several times slower than a warm one for its first seconds.
|
|
8
|
+
#
|
|
9
|
+
# Nothing reaches Redis: the commands are encoded and canned replies
|
|
10
|
+
# parsed by a connection that sends nothing. Only Robin's own middleware
|
|
11
|
+
# runs (the app's may have side effects), with jobs that use none of
|
|
12
|
+
# its features, and no job is performed.
|
|
13
|
+
module Warmup
|
|
14
|
+
ROUNDS = 500
|
|
15
|
+
|
|
16
|
+
# Every round pushes this many jobs in bulk, then this many again one
|
|
17
|
+
# at a time, and fetches and runs as many.
|
|
18
|
+
JOBS = 100
|
|
19
|
+
|
|
20
|
+
ARGS = [1, "two", {"three" => 3.0, "four" => [4]}].freeze
|
|
21
|
+
|
|
22
|
+
# A connection that encodes each round trip's commands and answers
|
|
23
|
+
# every one of them with +reply+ (RESP), parsed as a reply from Redis
|
|
24
|
+
# would be.
|
|
25
|
+
class Connection < Redis::Connection
|
|
26
|
+
# mutant:disable -- it never connects, so where to doesn't matter.
|
|
27
|
+
def initialize(reply)
|
|
28
|
+
super(host: "warmup")
|
|
29
|
+
@reply = reply
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
# mutant:disable -- the replies are all in the buffer, so reading
|
|
35
|
+
# them never waits (whatever the timeout).
|
|
36
|
+
def roundtrip(commands, _read_timeout)
|
|
37
|
+
encode(commands)
|
|
38
|
+
commands.each { @buf << @reply }
|
|
39
|
+
Array.new(commands.size) { read_reply(0) }
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# A pool of the one connection, for client middleware.
|
|
44
|
+
Pool = Struct.new(:connection)
|
|
45
|
+
|
|
46
|
+
class Pool
|
|
47
|
+
def with = yield(connection)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
class << self
|
|
51
|
+
def run(config, rounds: ROUNDS)
|
|
52
|
+
conn = Connection.new(":1\r\n")
|
|
53
|
+
# (A stand-in for a Redis::Pool: it has the #with the client uses.)
|
|
54
|
+
client = Client.new(config: config, pool: Pool.new(conn), chain: own(config.client_middleware)) # steep:ignore ArgumentTypeMismatch
|
|
55
|
+
server = own(config.server_middleware)
|
|
56
|
+
raw = Robin.dump_json(job(client))
|
|
57
|
+
fetches = Connection.new("*2\r\n$7\r\ndefault\r\n$#{raw.bytesize}\r\n#{raw}\r\n")
|
|
58
|
+
rounds.times do
|
|
59
|
+
push_bulk(client, conn)
|
|
60
|
+
JOBS.times { push(client, conn) }
|
|
61
|
+
JOBS.times { fetch_and_run(fetches, server, conn) }
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# A copy of +chain+ with only Robin's own middleware.
|
|
66
|
+
def own(chain)
|
|
67
|
+
chain = chain.dup
|
|
68
|
+
chain.each { |entry| chain.remove(entry.klass) unless entry.klass.to_s.start_with?("Robin::") }
|
|
69
|
+
chain
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def job(client) = client.normalize({"class" => "Robin::Warmup", "args" => ARGS})
|
|
75
|
+
|
|
76
|
+
def push_bulk(client, conn)
|
|
77
|
+
template = client.normalize({"class" => "Robin::Warmup", "args" => []})
|
|
78
|
+
payloads = Array.new(JOBS) { client.__send__(:run_middleware, client.__send__(:build, template, ARGS, nil, nil)) }
|
|
79
|
+
conn.pipelined { |p| client.queue_push(p, payloads) }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def push(client, conn)
|
|
83
|
+
payload = client.__send__(:invoke_middleware, job(client))
|
|
84
|
+
conn.pipelined { |p| client.queue_push(p, [payload]) }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# The job "runs" by being acknowledged.
|
|
88
|
+
def fetch_and_run(fetches, server, conn)
|
|
89
|
+
_queue, raw = fetches.call("BLMOVE", "queue", "inflight", "RIGHT", "LEFT", 2)
|
|
90
|
+
server.invoke(nil, Robin.load_json(raw), "default") { conn.call("DEL", "inflight") }
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|