robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
data/lib/robin/api.rb
ADDED
|
@@ -0,0 +1,777 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Read and manage everything Robin keeps in Redis. Every class takes an
|
|
5
|
+
# optional +config:+ so the web UI can point at several clusters.
|
|
6
|
+
#
|
|
7
|
+
# Robin::Stats.new.processed
|
|
8
|
+
# Robin::Queue.new("default").each { |job| job.delete if job.args == [42] }
|
|
9
|
+
# Robin::RetrySet.new.retry_all
|
|
10
|
+
# Robin::ProcessSet.new.each(&:quiet!)
|
|
11
|
+
class Stats
|
|
12
|
+
def initialize(config: Robin.config)
|
|
13
|
+
@config = config
|
|
14
|
+
fetch
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
attr_reader :processed, :failed, :scheduled_size, :retry_size, :dead_size,
|
|
18
|
+
:processes_size, :workers_size, :enqueued, :queues, :default_queue_latency
|
|
19
|
+
|
|
20
|
+
# Reads them all, in one round trip.
|
|
21
|
+
def fetch
|
|
22
|
+
c = @config
|
|
23
|
+
keys = [c.key("queues"), c.key("stat", "processed"), c.key("stat", "failed"), c.key("schedule"), c.key("retry"),
|
|
24
|
+
c.key("dead"), c.key("processes")]
|
|
25
|
+
processed, failed, @scheduled_size, @retry_size, @dead_size, oldest, queues, sizes, busy =
|
|
26
|
+
c.redis { |r| r.evalsha(Scripts::STATS, keys, ["#{c.key("queue")}:", c.key_prefix]) }
|
|
27
|
+
@processed = processed.to_i
|
|
28
|
+
@failed = failed.to_i
|
|
29
|
+
@default_queue_latency = Queue.latency_of(oldest)
|
|
30
|
+
@queues = queues.zip(sizes).sort_by { |q, n| [-n, q] }.to_h
|
|
31
|
+
@enqueued = @queues.values.sum
|
|
32
|
+
@processes_size = busy.size
|
|
33
|
+
@workers_size = busy.sum(&:to_i)
|
|
34
|
+
self
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def to_h
|
|
38
|
+
{"processed" => processed, "failed" => failed, "scheduled" => scheduled_size, "retries" => retry_size,
|
|
39
|
+
"dead" => dead_size, "processes" => processes_size, "busy" => workers_size, "enqueued" => enqueued,
|
|
40
|
+
"default_latency" => default_queue_latency}
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Daily processed/failed counts for the last +days+ days.
|
|
44
|
+
def history(days: 30, now: Time.now)
|
|
45
|
+
dates = (0...days).map { |i| (now.getutc - i * 86_400).strftime("%Y-%m-%d") }
|
|
46
|
+
c = @config
|
|
47
|
+
processed, failed = c.redis do |r|
|
|
48
|
+
r.pipelined do |p|
|
|
49
|
+
p.call("MGET", *dates.map { |d| c.key("stat", "processed", d) })
|
|
50
|
+
p.call("MGET", *dates.map { |d| c.key("stat", "failed", d) })
|
|
51
|
+
end
|
|
52
|
+
end #: Array[untyped]
|
|
53
|
+
{"processed" => dates.zip(processed.map(&:to_i)).to_h, "failed" => dates.zip(failed.map(&:to_i)).to_h}
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def reset(*names)
|
|
57
|
+
names = %w[processed failed] if names.empty?
|
|
58
|
+
@config.redis { |r| r.call("DEL", *names.map { |n| @config.key("stat", n) }) }
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
class Queue
|
|
63
|
+
include Enumerable
|
|
64
|
+
|
|
65
|
+
PAGE = 100
|
|
66
|
+
FIND_SLICE = 5_000
|
|
67
|
+
|
|
68
|
+
def self.all(config: Robin.config)
|
|
69
|
+
config.redis { |c| c.call("SMEMBERS", config.key("queues")) }.sort.map { |n| new(n, config: config) }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Seconds the oldest job in a queue has been waiting.
|
|
73
|
+
def self.latency_of(raw, now = Robin.now_ms)
|
|
74
|
+
return 0.0 unless raw
|
|
75
|
+
|
|
76
|
+
since = JobRecord.new(raw).waiting_since
|
|
77
|
+
# (since is an Integer or a Float; Steep picks neither of Integer#-'s overloads.)
|
|
78
|
+
since ? [(now - since) / 1000.0, 0.0].max : 0.0 # steep:ignore UnresolvedOverloading
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
attr_reader :name
|
|
82
|
+
|
|
83
|
+
def initialize(name = "default", config: Robin.config)
|
|
84
|
+
@name = name.to_s
|
|
85
|
+
@config = config
|
|
86
|
+
@key = config.key("queue", @name)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def size = @config.redis { |c| c.call("LLEN", @key) }
|
|
90
|
+
|
|
91
|
+
def latency = Queue.latency_of(@config.redis { |c| c.call("LINDEX", @key, -1) })
|
|
92
|
+
|
|
93
|
+
def paused? = @config.redis { |c| c.call("SISMEMBER", @config.key("paused"), name) } == 1
|
|
94
|
+
|
|
95
|
+
def pause! = @config.redis { |c| c.call("SADD", @config.key("paused"), name) }
|
|
96
|
+
|
|
97
|
+
def unpause! = @config.redis { |c| c.call("SREM", @config.key("paused"), name) }
|
|
98
|
+
|
|
99
|
+
def self.paused(config: Robin.config) = config.redis { |c| c.call("SMEMBERS", config.key("paused")) }
|
|
100
|
+
|
|
101
|
+
# Jobs from next-to-run to most recently pushed.
|
|
102
|
+
def each
|
|
103
|
+
return to_enum unless block_given?
|
|
104
|
+
|
|
105
|
+
offset = 0
|
|
106
|
+
last_size = nil #: Integer?
|
|
107
|
+
loop do
|
|
108
|
+
current, page = @config.redis do |c|
|
|
109
|
+
c.pipelined do |p|
|
|
110
|
+
p.call("LLEN", @key)
|
|
111
|
+
p.call(*page_command(offset))
|
|
112
|
+
end
|
|
113
|
+
end #: Array[untyped]
|
|
114
|
+
# Jobs removed since the last page (by the block, or by workers taking
|
|
115
|
+
# them) shift the tail; step back so none are skipped (reading the
|
|
116
|
+
# page again).
|
|
117
|
+
back = [last_size - current, offset].min if last_size
|
|
118
|
+
if back&.positive?
|
|
119
|
+
offset -= back
|
|
120
|
+
page = @config.redis { |c| c.call(*page_command(offset)) }
|
|
121
|
+
end
|
|
122
|
+
last_size = current
|
|
123
|
+
break if page.empty?
|
|
124
|
+
|
|
125
|
+
page.reverse_each { |raw| yield record(raw) }
|
|
126
|
+
offset += PAGE
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# A page of jobs: page 1 holds the jobs that will run next.
|
|
131
|
+
def page(number, per_page: 25)
|
|
132
|
+
start = -(number * per_page)
|
|
133
|
+
stop = -((number - 1) * per_page) - 1
|
|
134
|
+
raws = @config.redis { |c| c.call("LRANGE", @key, start, stop) }
|
|
135
|
+
raws.reverse.map { |raw| record(raw) }
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Finds a job by jid, searching the queue inside Redis a slice at a
|
|
139
|
+
# time (next to run first). Jobs moving while it searches may be missed.
|
|
140
|
+
def find_job(jid)
|
|
141
|
+
offset = 0
|
|
142
|
+
loop do
|
|
143
|
+
stop = -1 - offset
|
|
144
|
+
raw, length = @config.redis { |c| c.evalsha(Scripts::FIND_JID, [@key], [jid, stop - FIND_SLICE + 1, stop]) }
|
|
145
|
+
return record(raw) if raw
|
|
146
|
+
|
|
147
|
+
offset += FIND_SLICE
|
|
148
|
+
return if offset >= length
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Deletes every job of class +klass+ (as the dashboard names it, so an
|
|
153
|
+
# ActiveJob's own class), searching the queue inside Redis a slice at
|
|
154
|
+
# a time. Returns how many it deleted. Jobs moving while it searches
|
|
155
|
+
# may be missed.
|
|
156
|
+
def delete_by_class(klass)
|
|
157
|
+
name = klass.to_s
|
|
158
|
+
raise ArgumentError, "a class name is required" if name.empty?
|
|
159
|
+
|
|
160
|
+
deleted = offset = 0
|
|
161
|
+
loop do
|
|
162
|
+
stop = -1 - offset
|
|
163
|
+
removed, length = @config.redis { |c| c.evalsha(Scripts::DELETE_CLASS, [@key], [name, stop - FIND_SLICE + 1, stop]) }
|
|
164
|
+
deleted += removed
|
|
165
|
+
# What the slice lost, the jobs beyond it moved up to fill.
|
|
166
|
+
offset += FIND_SLICE - removed
|
|
167
|
+
return deleted if offset >= length
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# Deletes +jobs+ (JobRecords read from this queue), in one round trip;
|
|
172
|
+
# returns how many it deleted.
|
|
173
|
+
def delete_jobs(jobs)
|
|
174
|
+
@config.redis { |c| c.pipelined { |p| jobs.each { |job| p.call("LREM", @key, 1, job.value) } } }.sum
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def clear
|
|
178
|
+
@config.redis do |c|
|
|
179
|
+
c.multi do |tx|
|
|
180
|
+
tx.call("UNLINK", @key)
|
|
181
|
+
tx.call("SREM", @config.key("queues"), name)
|
|
182
|
+
tx.call("SREM", @config.key("paused"), name)
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
true
|
|
186
|
+
end
|
|
187
|
+
alias_method :delete, :clear
|
|
188
|
+
|
|
189
|
+
# In one round trip.
|
|
190
|
+
def to_h
|
|
191
|
+
size, oldest, paused = @config.redis do |c|
|
|
192
|
+
c.pipelined do |p|
|
|
193
|
+
p.call("LLEN", @key)
|
|
194
|
+
p.call("LINDEX", @key, -1)
|
|
195
|
+
p.call("SISMEMBER", @config.key("paused"), name)
|
|
196
|
+
end
|
|
197
|
+
end #: Array[untyped]
|
|
198
|
+
{"name" => name, "size" => size, "latency" => Queue.latency_of(oldest), "paused" => paused == 1}
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
private
|
|
202
|
+
|
|
203
|
+
# The list grows at the head; pages are read from the tail.
|
|
204
|
+
def page_command(offset)
|
|
205
|
+
stop = -1 - offset
|
|
206
|
+
["LRANGE", @key, stop - PAGE + 1, stop]
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# A job read from the queue (subclasses may return their own kind).
|
|
210
|
+
def record(raw) = JobRecord.new(raw, name, config: @config)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# A job as stored in Redis.
|
|
214
|
+
class JobRecord
|
|
215
|
+
attr_reader :item, :value, :queue
|
|
216
|
+
|
|
217
|
+
def initialize(value, queue = nil, config: Robin.config)
|
|
218
|
+
@config = config
|
|
219
|
+
if value.instance_of?(Hash)
|
|
220
|
+
@item = value
|
|
221
|
+
@value = Robin.dump_json(value)
|
|
222
|
+
else
|
|
223
|
+
@item = parse(value)
|
|
224
|
+
@value = value
|
|
225
|
+
end
|
|
226
|
+
@queue = queue || self["queue"]
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def [](key) = @item[key]
|
|
230
|
+
|
|
231
|
+
def klass = self["class"]
|
|
232
|
+
|
|
233
|
+
# The class users care about (unwraps ActiveJob).
|
|
234
|
+
def display_class = Robin.display_class(@item)
|
|
235
|
+
|
|
236
|
+
def args = self["args"]
|
|
237
|
+
|
|
238
|
+
def display_args
|
|
239
|
+
return Encryption.visible_args(@item) if self["encrypted"]
|
|
240
|
+
return args.first.fetch("arguments") if self["wrapped"] && args&.first.instance_of?(Hash)
|
|
241
|
+
|
|
242
|
+
args
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def jid = self["jid"]
|
|
246
|
+
|
|
247
|
+
def bid = self["bid"]
|
|
248
|
+
|
|
249
|
+
def tags = Array(self["tags"])
|
|
250
|
+
|
|
251
|
+
def error_class = self["error_class"]
|
|
252
|
+
|
|
253
|
+
def error_message = self["error_message"]
|
|
254
|
+
|
|
255
|
+
# The backtrace stored with the last failure (see Retrier.pack_backtrace).
|
|
256
|
+
def error_backtrace = Retrier.unpack_backtrace(self["error_backtrace"])
|
|
257
|
+
|
|
258
|
+
# How many times the job has failed (its retry_count counts from 0).
|
|
259
|
+
def failures = self["retry_count"] ? Integer(self["retry_count"]) + 1 : 0
|
|
260
|
+
|
|
261
|
+
def created_at = ms_time(self["created_at"])
|
|
262
|
+
|
|
263
|
+
def enqueued_at = ms_time(self["enqueued_at"])
|
|
264
|
+
|
|
265
|
+
def failed_at = ms_time(self["failed_at"])
|
|
266
|
+
|
|
267
|
+
def retried_at = ms_time(self["retried_at"])
|
|
268
|
+
|
|
269
|
+
# Epoch ms since when the job has been ready to run: the later of when
|
|
270
|
+
# it was pushed and when it was scheduled for.
|
|
271
|
+
def waiting_since = [Robin.ms_of(self["enqueued_at"]), self["at"] && Robin.epoch_ms(self["at"])].compact.max
|
|
272
|
+
|
|
273
|
+
def latency = Queue.latency_of(value)
|
|
274
|
+
|
|
275
|
+
def delete
|
|
276
|
+
@config.redis { |c| c.call("LREM", @config.key("queue", queue), 1, value) } == 1
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
def to_h
|
|
280
|
+
{"jid" => jid, "class" => klass, "display_class" => display_class, "args" => display_args, "queue" => queue,
|
|
281
|
+
"item" => (self["encrypted"] ? @item.merge({"args" => Encryption.visible_args(@item)}) : @item)}
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
private
|
|
285
|
+
|
|
286
|
+
def ms_time(time) = time && Time.at(Rational(Robin.ms_of(time), 1000))
|
|
287
|
+
|
|
288
|
+
def parse(raw)
|
|
289
|
+
Robin.load_json(raw)
|
|
290
|
+
rescue JSON::ParserError
|
|
291
|
+
{"class" => "<unparseable>", "args" => [], "raw" => raw}
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# A job in the schedule, retry or dead set.
|
|
296
|
+
class SortedEntry < JobRecord
|
|
297
|
+
attr_reader :score, :parent
|
|
298
|
+
|
|
299
|
+
def initialize(parent, score, value)
|
|
300
|
+
@parent = parent
|
|
301
|
+
@score = score.to_f
|
|
302
|
+
super(value, config: parent.config)
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# When the job is due (or, in the dead set, died).
|
|
306
|
+
def at = Time.at(Rational(Robin.epoch_ms(score), 1000))
|
|
307
|
+
|
|
308
|
+
def id = "#{score}-#{jid}"
|
|
309
|
+
|
|
310
|
+
def delete = parent.delete_entry(self)
|
|
311
|
+
|
|
312
|
+
# Push the job onto its queue now. If a client middleware raises, the
|
|
313
|
+
# job stays in the set.
|
|
314
|
+
def add_to_queue
|
|
315
|
+
remove_and { parent.pushing([self]) { Client.new(config: @config).push(queued_item) } }
|
|
316
|
+
end
|
|
317
|
+
alias_method :retry!, :add_to_queue
|
|
318
|
+
|
|
319
|
+
# The job without its time in the set, as it's pushed again or killed.
|
|
320
|
+
def bare_item = @item.except("at", "enqueued_at")
|
|
321
|
+
|
|
322
|
+
# The job #add_to_queue pushes.
|
|
323
|
+
def queued_item = bare_item
|
|
324
|
+
|
|
325
|
+
def reschedule(at)
|
|
326
|
+
score = Robin.epoch_seconds(Client.timestamp_ms(at))
|
|
327
|
+
@config.redis do |c|
|
|
328
|
+
c.multi do |tx|
|
|
329
|
+
tx.call("ZREM", parent.key, value)
|
|
330
|
+
tx.call("ZADD", parent.key, score, Robin.dump_json(@item.except("at")))
|
|
331
|
+
end
|
|
332
|
+
end
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
def kill!
|
|
336
|
+
remove_and { |item| DeadSet.new(config: @config).kill(Robin.dump_json(item)) }
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
def to_h = super.merge({"score" => score, "at" => score, "id" => id, "set" => parent.name})
|
|
340
|
+
|
|
341
|
+
private
|
|
342
|
+
|
|
343
|
+
# Remove first so concurrent actions can't duplicate the job.
|
|
344
|
+
def remove_and
|
|
345
|
+
return false unless @config.redis { |c| c.call("ZREM", parent.key, value) } == 1
|
|
346
|
+
|
|
347
|
+
yield bare_item
|
|
348
|
+
true
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
class JobSet
|
|
353
|
+
include Enumerable
|
|
354
|
+
|
|
355
|
+
PAGE = 100
|
|
356
|
+
|
|
357
|
+
attr_reader :name, :key, :config
|
|
358
|
+
|
|
359
|
+
def initialize(name, config: Robin.config)
|
|
360
|
+
@name = name
|
|
361
|
+
@config = config
|
|
362
|
+
@key = config.key(name)
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
def size = @config.redis { |c| c.call("ZCARD", key) }
|
|
366
|
+
|
|
367
|
+
def clear = @config.redis { |c| c.call("UNLINK", key) }
|
|
368
|
+
|
|
369
|
+
# Adds a job (a payload hash or its JSON) due at +at+ (anything
|
|
370
|
+
# Client.timestamp_ms takes).
|
|
371
|
+
def schedule(at, payload)
|
|
372
|
+
raw = payload.instance_of?(String) ? payload : Robin.dump_json(payload.except("at"))
|
|
373
|
+
@config.redis { |c| c.call("ZADD", key, Robin.epoch_seconds(Client.timestamp_ms(at)), raw) }
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
# Iterates by score: soonest first, or latest first with +reverse+.
|
|
377
|
+
def each(reverse: false)
|
|
378
|
+
return to_enum(:each, reverse: reverse) unless block_given?
|
|
379
|
+
|
|
380
|
+
offset = 0
|
|
381
|
+
last_size = nil #: Integer?
|
|
382
|
+
loop do
|
|
383
|
+
current, raw = @config.redis do |c|
|
|
384
|
+
c.pipelined do |p|
|
|
385
|
+
p.call("ZCARD", key)
|
|
386
|
+
p.call(*range_command(offset, PAGE, reverse))
|
|
387
|
+
end
|
|
388
|
+
end #: Array[untyped]
|
|
389
|
+
# Entries removed since the last page (e.g. by the block) shift the
|
|
390
|
+
# ranks; step back so none are skipped (reading the page again).
|
|
391
|
+
back = [last_size - current, offset].min if last_size
|
|
392
|
+
if back&.positive?
|
|
393
|
+
offset -= back
|
|
394
|
+
raw = @config.redis { |c| c.call(*range_command(offset, PAGE, reverse)) }
|
|
395
|
+
end
|
|
396
|
+
last_size = current
|
|
397
|
+
entries = entries_from(raw)
|
|
398
|
+
break if entries.empty?
|
|
399
|
+
|
|
400
|
+
entries.each { |entry| yield entry }
|
|
401
|
+
offset += PAGE
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def page(number, per_page: 25, reverse: false)
|
|
406
|
+
range((number - 1) * per_page, per_page, reverse)
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
# A #page and the set's size, in one round trip.
|
|
410
|
+
def page_and_size(number, per_page:, reverse:)
|
|
411
|
+
raw, size = @config.redis do |c|
|
|
412
|
+
c.pipelined do |p|
|
|
413
|
+
p.call(*range_command((number - 1) * per_page, per_page, reverse))
|
|
414
|
+
p.call("ZCARD", key)
|
|
415
|
+
end
|
|
416
|
+
end #: Array[untyped]
|
|
417
|
+
[entries_from(raw), size]
|
|
418
|
+
end
|
|
419
|
+
|
|
420
|
+
# Finds entries whose JSON contains +term+ using ZSCAN (server-side
|
|
421
|
+
# glob match, so it's fast even for large sets). Returns up to +limit+
|
|
422
|
+
# matches, soonest first.
|
|
423
|
+
def scan(term, limit: 500)
|
|
424
|
+
return [] if term.to_s.empty?
|
|
425
|
+
|
|
426
|
+
pattern = "*#{term.to_s.gsub(/[\\*?\[\]]/) { |m| "\\#{m}" }}*"
|
|
427
|
+
results = [] #: Array[SortedEntry]
|
|
428
|
+
cursor = "0"
|
|
429
|
+
@config.redis do |c|
|
|
430
|
+
loop do
|
|
431
|
+
cursor, pairs = c.call("ZSCAN", key, cursor, "MATCH", pattern, "COUNT", 1000)
|
|
432
|
+
pairs.each_slice(2) { |value, score| results << entry(score, value) }
|
|
433
|
+
break if cursor == "0" || results.size >= limit
|
|
434
|
+
end
|
|
435
|
+
end
|
|
436
|
+
results.first(limit).sort_by(&:score)
|
|
437
|
+
end
|
|
438
|
+
|
|
439
|
+
def find_job(jid)
|
|
440
|
+
scan(jid).find { |e| e.jid == jid }
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# Fetch entries with exactly this score (and optionally jid).
|
|
444
|
+
def fetch(score, jid = nil)
|
|
445
|
+
raw = @config.redis { |c| c.call("ZRANGE", key, score, score, "BYSCORE", "WITHSCORES") }
|
|
446
|
+
entries = raw.each_slice(2).map { |value, s| entry(s, value) }
|
|
447
|
+
jid ? entries.select { |e| e.jid == jid } : entries
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
# Look up by the "<score>-<jid>" ids used in the web UI.
|
|
451
|
+
def find_by_id(id)
|
|
452
|
+
score, jid = id.split("-", 2)
|
|
453
|
+
fetch(score, jid).first
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def delete_entry(entry)
|
|
457
|
+
@config.redis { |c| c.call("ZREM", key, entry.value) } == 1
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
def delete_by_jid(score, jid)
|
|
461
|
+
fetch(score, jid).each(&:delete).any?
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
# Looks up several "<score>-<jid>" ids in one round trip; returns the
|
|
465
|
+
# entries found.
|
|
466
|
+
def find_all_by_id(ids)
|
|
467
|
+
wanted = ids.map { |id| id.split("-", 2) }
|
|
468
|
+
rows = @config.redis { |c| c.pipelined { |p| wanted.each { |score, _jid| p.call("ZRANGE", key, score, score, "BYSCORE", "WITHSCORES") } } }
|
|
469
|
+
wanted.zip(rows).flat_map do |(_score, jid), raw|
|
|
470
|
+
raw.each_slice(2).map { |value, score| entry(score, value) }.select { |e| e.jid == jid }
|
|
471
|
+
end
|
|
472
|
+
end
|
|
473
|
+
|
|
474
|
+
# Removes +entries+ (the ones still in the set), in one round trip,
|
|
475
|
+
# and returns those it removed: removed first, so concurrent actions
|
|
476
|
+
# can't duplicate a job.
|
|
477
|
+
def take(entries)
|
|
478
|
+
removed = @config.redis { |c| c.pipelined { |p| entries.each { |e| p.call("ZREM", key, e.value) } } }
|
|
479
|
+
entries.zip(removed).filter_map { |entry, gone| entry if gone == 1 }
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
# Deletes +entries+ in one round trip; returns how many.
|
|
483
|
+
def delete_entries(entries) = take(entries).size
|
|
484
|
+
|
|
485
|
+
# Pushes +entries+ onto their queues now (as SortedEntry#add_to_queue
|
|
486
|
+
# does), in two round trips; returns how many.
|
|
487
|
+
def retry_entries(entries)
|
|
488
|
+
taken = take(entries)
|
|
489
|
+
pushing(taken) { taken.each { |entry| Client.new(config: @config).push(entry.queued_item) } }
|
|
490
|
+
taken.size
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
# Runs the block's pushes of +taken+ (entries already removed from the
|
|
494
|
+
# set) together, sent once it ends. If the block raises (a client
|
|
495
|
+
# middleware, say), nothing was sent, and +taken+ go back in the set.
|
|
496
|
+
# An error sending them doesn't put them back: they may have reached
|
|
497
|
+
# their queues all the same, and a job mustn't run twice.
|
|
498
|
+
def pushing(taken)
|
|
499
|
+
Client.together do
|
|
500
|
+
yield
|
|
501
|
+
rescue
|
|
502
|
+
@config.redis { |c| c.call("ZADD", key, *taken.flat_map { |entry| [entry.score, entry.value] }) }
|
|
503
|
+
raise
|
|
504
|
+
end
|
|
505
|
+
end
|
|
506
|
+
|
|
507
|
+
# Moves +entries+ to the dead set (as SortedEntry#kill! does), in two
|
|
508
|
+
# round trips; returns how many.
|
|
509
|
+
def kill_entries(entries)
|
|
510
|
+
taken = take(entries)
|
|
511
|
+
dead = DeadSet.new(config: @config)
|
|
512
|
+
@config.redis { |c| c.multi { |tx| taken.each { |e| dead.queue_kill(tx, Robin.dump_json(e.bare_item), nil) } } } unless taken.empty?
|
|
513
|
+
taken.size
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
# Yields the entries that are in the set now, soonest first, to a
|
|
517
|
+
# block that removes each one (retry!, kill!, delete) and returns
|
|
518
|
+
# truthy if it did. Unlike #each, this never holds the whole set in
|
|
519
|
+
# memory, and it ignores entries added meanwhile. Returns how many the
|
|
520
|
+
# block removed.
|
|
521
|
+
def drain(&) = drain_pages { |page| page.count(&) }
|
|
522
|
+
|
|
523
|
+
# Like #drain, a page at a time: the block gets a page of entries,
|
|
524
|
+
# removes them (e.g. with #retry_entries) and returns how many it did.
|
|
525
|
+
def drain_pages
|
|
526
|
+
removed = 0
|
|
527
|
+
kept = 0
|
|
528
|
+
remaining = size
|
|
529
|
+
while remaining.positive?
|
|
530
|
+
page = range(kept, [PAGE, remaining].min, false)
|
|
531
|
+
break if page.empty?
|
|
532
|
+
|
|
533
|
+
done = yield(page)
|
|
534
|
+
removed += done
|
|
535
|
+
kept += page.size - done
|
|
536
|
+
remaining -= page.size
|
|
537
|
+
end
|
|
538
|
+
removed
|
|
539
|
+
end
|
|
540
|
+
|
|
541
|
+
private
|
|
542
|
+
|
|
543
|
+
def range(start, count, reverse) = entries_from(@config.redis { |c| c.call(*range_command(start, count, reverse)) })
|
|
544
|
+
|
|
545
|
+
def range_command(start, count, reverse) = ["ZRANGE", key, start, start + count - 1, *("REV" if reverse), "WITHSCORES"]
|
|
546
|
+
|
|
547
|
+
def entries_from(raw) = raw.each_slice(2).map { |value, score| entry(score, value) }
|
|
548
|
+
|
|
549
|
+
# An entry read from the set (subclasses may return their own kind).
|
|
550
|
+
def entry(score, value) = SortedEntry.new(self, score, value)
|
|
551
|
+
end
|
|
552
|
+
|
|
553
|
+
class ScheduledSet < JobSet
|
|
554
|
+
def initialize(config: Robin.config) = super("schedule", config: config)
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
class RetrySet < JobSet
|
|
558
|
+
def initialize(config: Robin.config) = super("retry", config: config)
|
|
559
|
+
|
|
560
|
+
# Pushes every entry back onto its queue, a page at a time (three
|
|
561
|
+
# round trips per page); returns how many.
|
|
562
|
+
def retry_all = drain_pages { |page| retry_entries(page) }
|
|
563
|
+
|
|
564
|
+
# Moves every entry to the dead set, a page at a time; returns how
|
|
565
|
+
# many.
|
|
566
|
+
def kill_all = drain_pages { |page| kill_entries(page) }
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
class DeadSet < JobSet
|
|
570
|
+
def initialize(config: Robin.config) = super("dead", config: config)
|
|
571
|
+
|
|
572
|
+
# Adds a payload to the dead set (trimming it by age and size), and
|
|
573
|
+
# optionally acknowledges the unit of work in the same transaction.
|
|
574
|
+
def kill(raw, ack: nil, notify: false)
|
|
575
|
+
@config.redis { |c| c.multi { |tx| queue_kill(tx, raw, ack) } }
|
|
576
|
+
if notify
|
|
577
|
+
payload = Robin.load_json(raw)
|
|
578
|
+
@config.death_handlers.each { |h| h.call(payload, nil) }
|
|
579
|
+
end
|
|
580
|
+
true
|
|
581
|
+
end
|
|
582
|
+
|
|
583
|
+
def retry_all = drain_pages { |page| retry_entries(page) }
|
|
584
|
+
|
|
585
|
+
# Queues what #kill sends into +tx+ (a transaction), for a caller that
|
|
586
|
+
# adds its own commands to it.
|
|
587
|
+
def queue_kill(tx, raw, ack)
|
|
588
|
+
now = Robin.epoch_seconds(Robin.now_ms)
|
|
589
|
+
tx.call("ZADD", key, now, raw)
|
|
590
|
+
tx.call("ZREMRANGEBYSCORE", key, "-inf", now - @config.dead_timeout)
|
|
591
|
+
tx.call("ZREMRANGEBYRANK", key, 0, -(@config.dead_max_jobs + 1))
|
|
592
|
+
ack&.acknowledge(tx)
|
|
593
|
+
end
|
|
594
|
+
end
|
|
595
|
+
|
|
596
|
+
class ProcessRecord
|
|
597
|
+
attr_reader :identity, :info, :busy, :beat, :rss, :rtt_us
|
|
598
|
+
|
|
599
|
+
# GVL contention over its last heartbeat interval, as a percentage
|
|
600
|
+
# (see Robin::GVL), or nil where it isn't measured.
|
|
601
|
+
attr_reader :gvl_contention
|
|
602
|
+
|
|
603
|
+
def initialize(identity, data, config: Robin.config)
|
|
604
|
+
@config = config
|
|
605
|
+
@identity = identity
|
|
606
|
+
@info = Robin.load_json(data["info"] || "{}")
|
|
607
|
+
@busy = data["busy"].to_i
|
|
608
|
+
@beat = Robin.epoch_ms(data["beat"] || 0)
|
|
609
|
+
@quiet = data["quiet"] == "true"
|
|
610
|
+
@leader = data["leader"] == "1"
|
|
611
|
+
@rss = data["rss"].to_i
|
|
612
|
+
@rtt_us = data["rtt_us"].to_i
|
|
613
|
+
@gvl_contention = data["gvl"]&.to_f
|
|
614
|
+
end
|
|
615
|
+
|
|
616
|
+
def [](key) = @info[key]
|
|
617
|
+
|
|
618
|
+
# Epoch ms (the record keeps epoch seconds).
|
|
619
|
+
def started_at = (at = self["started_at"]) && Robin.epoch_ms(at)
|
|
620
|
+
|
|
621
|
+
def quiet? = @quiet
|
|
622
|
+
|
|
623
|
+
def leader? = @leader
|
|
624
|
+
|
|
625
|
+
def quiet! = signal("quiet")
|
|
626
|
+
|
|
627
|
+
def stop! = signal("stop")
|
|
628
|
+
|
|
629
|
+
def dump_threads = signal("dump")
|
|
630
|
+
|
|
631
|
+
# Has the process run +count+ workers in capsule +capsule+ from its
|
|
632
|
+
# next heartbeat on: new ones start, and ones no longer needed finish
|
|
633
|
+
# their current job first. A process whose Redis pool was given a size
|
|
634
|
+
# (:pool_size) too small for them logs a warning and changes nothing.
|
|
635
|
+
def resize!(count, capsule: "default")
|
|
636
|
+
raise ArgumentError, "a capsule needs a concurrency above 0, not #{count.inspect}" unless count.instance_of?(Integer) && count.positive?
|
|
637
|
+
|
|
638
|
+
send_signal("RESIZE #{count} #{capsule}")
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
# With times as epoch ms, and each capsule's queue weights as
|
|
642
|
+
# "queues".
|
|
643
|
+
def to_h
|
|
644
|
+
capsules = @info["capsules"]&.transform_values { |c| {"concurrency" => c["concurrency"], "queues" => c["weights"]} }
|
|
645
|
+
@info.merge({"identity" => identity, "started_at" => started_at, "capsules" => capsules, "busy" => busy,
|
|
646
|
+
"beat" => beat, "quiet" => quiet?, "leader" => leader?, "rss_kb" => rss, "rtt_us" => rtt_us,
|
|
647
|
+
"gvl_contention" => gvl_contention}).compact
|
|
648
|
+
end
|
|
649
|
+
|
|
650
|
+
private
|
|
651
|
+
|
|
652
|
+
# Remote signals are named like the Unix signals that do the same.
|
|
653
|
+
SIGNALS = {"quiet" => "TSTP", "stop" => "TERM", "dump" => "TTIN"}.freeze
|
|
654
|
+
|
|
655
|
+
def signal(name) = send_signal(SIGNALS.fetch(name))
|
|
656
|
+
|
|
657
|
+
def send_signal(signal)
|
|
658
|
+
key = "#{@config.process_key(identity)}-signals"
|
|
659
|
+
@config.redis do |c|
|
|
660
|
+
c.pipelined do |p|
|
|
661
|
+
p.call("LPUSH", key, signal)
|
|
662
|
+
p.call("EXPIRE", key, 60)
|
|
663
|
+
end
|
|
664
|
+
end
|
|
665
|
+
true
|
|
666
|
+
end
|
|
667
|
+
end
|
|
668
|
+
|
|
669
|
+
class ProcessSet
|
|
670
|
+
include Enumerable
|
|
671
|
+
|
|
672
|
+
def initialize(config: Robin.config)
|
|
673
|
+
@config = config
|
|
674
|
+
end
|
|
675
|
+
|
|
676
|
+
def each(&block)
|
|
677
|
+
return to_enum unless block
|
|
678
|
+
|
|
679
|
+
Scripts.member_hashes(@config, @config.key("processes"), @config.key_prefix).each do |identity, data|
|
|
680
|
+
# A process that died recently may still be listed without a hash.
|
|
681
|
+
yield record(identity, data) unless data.empty?
|
|
682
|
+
end
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
def size = count
|
|
686
|
+
|
|
687
|
+
def total_concurrency = sum { |p| p["concurrency"].to_i }
|
|
688
|
+
|
|
689
|
+
def total_rss_kb = sum(&:rss)
|
|
690
|
+
|
|
691
|
+
def leader = Leader.current(@config)
|
|
692
|
+
|
|
693
|
+
private
|
|
694
|
+
|
|
695
|
+
# A process read from Redis (subclasses may return their own kind).
|
|
696
|
+
def record(identity, data) = ProcessRecord.new(identity, data, config: @config)
|
|
697
|
+
end
|
|
698
|
+
|
|
699
|
+
# Jobs currently running across the cluster.
|
|
700
|
+
# Finds where a job is by jid: running, in the schedule, retry or dead
|
|
701
|
+
# set, or (only if none of those has it) waiting in a queue.
|
|
702
|
+
#
|
|
703
|
+
# Robin::JobLocator.new.locate(jid)
|
|
704
|
+
# # => [{"where" => "retries", "job" => #<Robin::SortedEntry>}]
|
|
705
|
+
#
|
|
706
|
+
# With Config#jid_index on, the queue the job was pushed to is searched
|
|
707
|
+
# first, and is the only queue searched when it's larger than
|
|
708
|
+
# QUEUE_LIMIT.
|
|
709
|
+
class JobLocator
|
|
710
|
+
SETS = {"scheduled" => ScheduledSet, "retries" => RetrySet, "dead" => DeadSet}.freeze
|
|
711
|
+
QUEUE_LIMIT = 1_000_000
|
|
712
|
+
|
|
713
|
+
def initialize(config: Robin.config)
|
|
714
|
+
@config = config
|
|
715
|
+
end
|
|
716
|
+
|
|
717
|
+
def locate(jid)
|
|
718
|
+
found = busy(jid) + sets(jid)
|
|
719
|
+
found.empty? ? queued(jid) : found
|
|
720
|
+
end
|
|
721
|
+
|
|
722
|
+
private
|
|
723
|
+
|
|
724
|
+
def busy(jid)
|
|
725
|
+
WorkSet.new(config: @config).filter_map do |identity, tid, work|
|
|
726
|
+
payload = work.fetch("payload")
|
|
727
|
+
next unless payload.fetch("jid") == jid
|
|
728
|
+
|
|
729
|
+
{"where" => "busy", "identity" => identity, "thread" => tid,
|
|
730
|
+
"job" => JobRecord.new(payload, work.fetch("queue"), config: @config)}
|
|
731
|
+
end
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
def sets(jid)
|
|
735
|
+
SETS.filter_map do |where, klass|
|
|
736
|
+
entry = klass.new(config: @config).find_job(jid)
|
|
737
|
+
{"where" => where, "job" => entry} if entry
|
|
738
|
+
end
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
def queued(jid)
|
|
742
|
+
hint = @config.redis { |c| c.call("GET", @config.key("jid", jid)) }
|
|
743
|
+
hinted, others = Queue.all(config: @config).partition { |q| q.name == hint }
|
|
744
|
+
(hinted + others.reject { |q| q.size > QUEUE_LIMIT }).each do |queue|
|
|
745
|
+
job = queue.find_job(jid)
|
|
746
|
+
return [{"where" => "queue", "queue" => queue.name, "job" => job}] if job
|
|
747
|
+
end
|
|
748
|
+
[]
|
|
749
|
+
end
|
|
750
|
+
end
|
|
751
|
+
|
|
752
|
+
class WorkSet
|
|
753
|
+
include Enumerable
|
|
754
|
+
|
|
755
|
+
def initialize(config: Robin.config)
|
|
756
|
+
@config = config
|
|
757
|
+
end
|
|
758
|
+
|
|
759
|
+
# Yields (identity, thread name, {"queue", "payload" (the job hash),
|
|
760
|
+
# "run_at" (epoch ms)}).
|
|
761
|
+
def each
|
|
762
|
+
return to_enum unless block_given?
|
|
763
|
+
|
|
764
|
+
Scripts.member_hashes(@config, @config.key("processes"), @config.key_prefix, ":work").each do |identity, work|
|
|
765
|
+
# Each process's jobs by thread, in the same order everywhere.
|
|
766
|
+
work.sort_by(&:first).each do |tid, json|
|
|
767
|
+
work = Robin.load_json(json)
|
|
768
|
+
payload = work.fetch("payload")
|
|
769
|
+
yield identity, tid, {"queue" => work.fetch("queue"), "run_at" => Robin.epoch_ms(work.fetch("run_at")),
|
|
770
|
+
"payload" => payload.instance_of?(String) ? Robin.load_json(payload) : payload}
|
|
771
|
+
end
|
|
772
|
+
end
|
|
773
|
+
end
|
|
774
|
+
|
|
775
|
+
def size = count
|
|
776
|
+
end
|
|
777
|
+
end
|