robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Moves due jobs from the schedule and retry sets onto their queues.
|
|
5
|
+
#
|
|
6
|
+
# Every server process runs one. The poll interval is randomized and
|
|
7
|
+
# scaled by the number of live processes, so the cluster as a whole
|
|
8
|
+
# checks about once every +poll_interval+ seconds no matter how many
|
|
9
|
+
# processes there are.
|
|
10
|
+
#
|
|
11
|
+
# Each poll also learns when the next job is due. The cluster's leader
|
|
12
|
+
# wakes up for it, if that's sooner than its next poll, so jobs known
|
|
13
|
+
# ahead start on time rather than up to a poll late. Only the leader
|
|
14
|
+
# does, so the cluster doesn't wake up as one. Jobs scheduled since the
|
|
15
|
+
# last poll for sooner than the next known one wait for a poll.
|
|
16
|
+
class Scheduler
|
|
17
|
+
BATCH = 500
|
|
18
|
+
SETS = %w[schedule retry].freeze
|
|
19
|
+
|
|
20
|
+
# The least the leader sleeps between polls (seconds), so a steady
|
|
21
|
+
# stream of jobs coming due costs at most 100 polls a second.
|
|
22
|
+
MIN_WAIT = 0.01
|
|
23
|
+
|
|
24
|
+
def initialize(config)
|
|
25
|
+
@config = config
|
|
26
|
+
@mutex = Mutex.new
|
|
27
|
+
@cond = ConditionVariable.new
|
|
28
|
+
# Each set's earliest score (epoch seconds) as of its last poll, or
|
|
29
|
+
# nil if it was empty.
|
|
30
|
+
@next = {}
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def start
|
|
34
|
+
@thread = Thread.new do
|
|
35
|
+
Thread.current.name = "robin.scheduler"
|
|
36
|
+
sleep_for(rand * @config.poll_interval) # stagger process startup
|
|
37
|
+
until @done
|
|
38
|
+
begin
|
|
39
|
+
enqueue_due
|
|
40
|
+
rescue => e
|
|
41
|
+
@config.handle_exception(e, {context: "scheduler"})
|
|
42
|
+
end
|
|
43
|
+
sleep_for(interval)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
self
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def stop
|
|
50
|
+
@done = true
|
|
51
|
+
@mutex.synchronize { @cond.signal }
|
|
52
|
+
@thread&.join(2)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Enqueues all currently due jobs. Returns the number moved.
|
|
56
|
+
#
|
|
57
|
+
# One round trip covers both sets (and counts the live processes for
|
|
58
|
+
# #interval); a set that filled a whole batch is drained with more.
|
|
59
|
+
def enqueue_due(now = Robin.now_ms)
|
|
60
|
+
@config.redis do |conn|
|
|
61
|
+
sets = SETS
|
|
62
|
+
total = 0
|
|
63
|
+
until sets.empty?
|
|
64
|
+
*replies, @processes = conn.pipelined do |p|
|
|
65
|
+
sets.each do |set|
|
|
66
|
+
p.evalsha(Scripts::ENQUEUE_DUE, [@config.key(set), @config.key("queues")],
|
|
67
|
+
[Robin.epoch_seconds(now), "#{@config.key("queue")}:", BATCH, now])
|
|
68
|
+
end
|
|
69
|
+
p.call("SCARD", @config.key("processes"))
|
|
70
|
+
end
|
|
71
|
+
sets.zip(replies) { |set, (_, score)| @next[set] = score && Float(score) }
|
|
72
|
+
moved = replies.map { |count, _score| count }
|
|
73
|
+
total += moved.sum
|
|
74
|
+
sets = @done ? [] : sets.zip(moved).filter_map { |set, n| set if n.equal?(BATCH) }
|
|
75
|
+
end
|
|
76
|
+
total
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# When the next job of either set is due (epoch seconds), as of the
|
|
81
|
+
# last poll; nil if both were empty.
|
|
82
|
+
def next_due = @next.values.compact.min
|
|
83
|
+
|
|
84
|
+
# Seconds until the next poll: about +poll_interval+ times the live
|
|
85
|
+
# processes as last counted by #enqueue_due (one if it never got an
|
|
86
|
+
# answer), or, for the leader, until the next job is due if sooner.
|
|
87
|
+
def interval
|
|
88
|
+
poll = @config.poll_interval * [@processes.to_i, 1].max * (0.5 + rand)
|
|
89
|
+
due = next_due
|
|
90
|
+
return poll unless due && Leader.leader?(@config)
|
|
91
|
+
|
|
92
|
+
wait = [due - Robin.epoch_seconds(Robin.now_ms), MIN_WAIT].max
|
|
93
|
+
[wait, poll].min
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
private
|
|
97
|
+
|
|
98
|
+
def sleep_for(seconds)
|
|
99
|
+
@mutex.synchronize { @cond.wait(@mutex, seconds) unless @done }
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Robin
|
|
4
|
+
# Lua scripts used on hot paths. Each runs atomically on the server.
|
|
5
|
+
module Scripts
|
|
6
|
+
# Acknowledges finished jobs (deletes their in-flight keys), then
|
|
7
|
+
# moves up to +count+ jobs into in-flight keys, one each, taking from
|
|
8
|
+
# the first non-paused queue until it runs dry before moving on to the
|
|
9
|
+
# next: one round trip for several jobs.
|
|
10
|
+
#
|
|
11
|
+
# KEYS: paused set, +count+ in-flight keys to fill, queue lists, then
|
|
12
|
+
# the finished jobs' in-flight keys
|
|
13
|
+
# ARGV: count, the queue names (same order as the queue list keys)
|
|
14
|
+
# Returns: a flat list of queue name, payload pairs, in the order of
|
|
15
|
+
# the in-flight keys they went to
|
|
16
|
+
FETCH = Redis::Script.new(<<~LUA)
|
|
17
|
+
local count = tonumber(ARGV[1])
|
|
18
|
+
local acks = count + #ARGV + 1
|
|
19
|
+
if acks <= #KEYS then redis.call('DEL', unpack(KEYS, acks)) end
|
|
20
|
+
local taken = {}
|
|
21
|
+
local filled = 0
|
|
22
|
+
for i = 2, #ARGV do
|
|
23
|
+
local name = ARGV[i]
|
|
24
|
+
if redis.call('SISMEMBER', KEYS[1], name) == 0 then
|
|
25
|
+
while filled < count do
|
|
26
|
+
local job = redis.call('LMOVE', KEYS[count + i], KEYS[filled + 2], 'RIGHT', 'LEFT')
|
|
27
|
+
if not job then break end
|
|
28
|
+
filled = filled + 1
|
|
29
|
+
taken[#taken + 1] = name
|
|
30
|
+
taken[#taken + 1] = job
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
if filled == count then break end
|
|
34
|
+
end
|
|
35
|
+
return taken
|
|
36
|
+
LUA
|
|
37
|
+
|
|
38
|
+
# Moves every due job from a sorted set (schedule or retry) onto its
|
|
39
|
+
# queue, recording when as its enqueued_at. Stored jobs have none, so
|
|
40
|
+
# it's appended to the JSON rather than re-encoding it, which could
|
|
41
|
+
# change how numbers in it are written (were there one, the last
|
|
42
|
+
# occurrence wins when it's parsed).
|
|
43
|
+
#
|
|
44
|
+
# Returns how many jobs it took (at most the max; some may have been
|
|
45
|
+
# taken by another process meanwhile) and, if any job is left in the
|
|
46
|
+
# set, the earliest one's score: {count[, score]}.
|
|
47
|
+
#
|
|
48
|
+
# KEYS: sorted set, queues set
|
|
49
|
+
# ARGV: now (epoch seconds), queue key prefix ("<prefix>queue:"), max
|
|
50
|
+
# jobs per call, now (epoch ms)
|
|
51
|
+
ENQUEUE_DUE = Redis::Script.new(<<~LUA)
|
|
52
|
+
local jobs = redis.call('ZRANGE', KEYS[1], '-inf', ARGV[1], 'BYSCORE', 'LIMIT', 0, tonumber(ARGV[3]))
|
|
53
|
+
for _, job in ipairs(jobs) do
|
|
54
|
+
if redis.call('ZREM', KEYS[1], job) == 1 then
|
|
55
|
+
local ok, decoded = pcall(cjson.decode, job)
|
|
56
|
+
local queue = 'default'
|
|
57
|
+
if ok and type(decoded) == 'table' then
|
|
58
|
+
if type(decoded['queue']) == 'string' then queue = decoded['queue'] end
|
|
59
|
+
-- Only a non-empty JSON object ("{...}") gets the field.
|
|
60
|
+
if string.byte(job, 1) == 123 and next(decoded) then
|
|
61
|
+
job = string.sub(job, 1, -2) .. ',"enqueued_at":' .. ARGV[4] .. '}'
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
redis.call('SADD', KEYS[2], queue)
|
|
65
|
+
redis.call('LPUSH', ARGV[2] .. queue, job)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
return {#jobs, redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')[2]}
|
|
69
|
+
LUA
|
|
70
|
+
|
|
71
|
+
# Moves the jobs in a process's in-flight keys back onto the front of
|
|
72
|
+
# their queues, oldest first, unregistering each key once it's empty.
|
|
73
|
+
# Returns what it did to each job: status, job, status, job...
|
|
74
|
+
# ("requeued" or "dead").
|
|
75
|
+
#
|
|
76
|
+
# When counting (crash recovery, not a graceful shutdown), each
|
|
77
|
+
# recovery of a job bumps its orphan:<jid> counter, and a job orphaned
|
|
78
|
+
# too many times (a "poison pill" that keeps crashing processes) goes
|
|
79
|
+
# to the dead set instead. A counted job gets a "recovered" field so
|
|
80
|
+
# the processor knows to clear its counter once it succeeds.
|
|
81
|
+
#
|
|
82
|
+
# KEYS: the process's in-flight set (or, from before in-flight sets,
|
|
83
|
+
# the one list it kept its jobs in), queues set, dead set
|
|
84
|
+
# ARGV: queue key prefix ("<prefix>queue:"), max recoveries, now (epoch
|
|
85
|
+
# seconds, the dead set's score), count ("true" or "false"), orphan
|
|
86
|
+
# counter key prefix
|
|
87
|
+
RECOVER = Redis::Script.new(<<~LUA)
|
|
88
|
+
local done = {}
|
|
89
|
+
local function drain(key)
|
|
90
|
+
while true do
|
|
91
|
+
local job = redis.call('RPOP', key)
|
|
92
|
+
if not job then return end
|
|
93
|
+
local ok, decoded = pcall(cjson.decode, job)
|
|
94
|
+
local queue, jid = 'default', nil
|
|
95
|
+
if ok and type(decoded) == 'table' then
|
|
96
|
+
if type(decoded['queue']) == 'string' then queue = decoded['queue'] end
|
|
97
|
+
jid = decoded['jid']
|
|
98
|
+
end
|
|
99
|
+
local status = 'requeued'
|
|
100
|
+
if jid and ARGV[4] == 'true' then
|
|
101
|
+
local key = ARGV[5] .. jid
|
|
102
|
+
local n = redis.call('INCR', key)
|
|
103
|
+
redis.call('EXPIRE', key, 604800)
|
|
104
|
+
if n > tonumber(ARGV[2]) then
|
|
105
|
+
redis.call('ZADD', KEYS[3], ARGV[3], job)
|
|
106
|
+
status = 'dead'
|
|
107
|
+
-- Splice the field in rather than re-encoding, which could
|
|
108
|
+
-- change how numbers in the payload are written.
|
|
109
|
+
elseif decoded['recovered'] == nil then
|
|
110
|
+
job = '{"recovered":true,' .. string.sub(job, 2)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
if status == 'requeued' then
|
|
114
|
+
redis.call('SADD', KEYS[2], queue)
|
|
115
|
+
redis.call('RPUSH', ARGV[1] .. queue, job)
|
|
116
|
+
end
|
|
117
|
+
done[#done + 1] = status
|
|
118
|
+
done[#done + 1] = job
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
if redis.call('TYPE', KEYS[1])['ok'] == 'list' then
|
|
122
|
+
drain(KEYS[1])
|
|
123
|
+
else
|
|
124
|
+
for _, key in ipairs(redis.call('SMEMBERS', KEYS[1])) do
|
|
125
|
+
drain(key)
|
|
126
|
+
redis.call('SREM', KEYS[1], key)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
return done
|
|
130
|
+
LUA
|
|
131
|
+
|
|
132
|
+
# Forgets a dead process once its jobs are recovered: unregisters its
|
|
133
|
+
# in-flight set and stopping marker, unless the set filled again (the
|
|
134
|
+
# process only looked dead; its heartbeat had lapsed), and its
|
|
135
|
+
# identity.
|
|
136
|
+
#
|
|
137
|
+
# KEYS: its in-flight set, the in-flight sets' set, its stopping
|
|
138
|
+
# marker, the processes set
|
|
139
|
+
# ARGV: its identity
|
|
140
|
+
FORGET = Redis::Script.new(<<~LUA)
|
|
141
|
+
if redis.call('EXISTS', KEYS[1]) == 0 then
|
|
142
|
+
redis.call('SREM', KEYS[2], KEYS[1])
|
|
143
|
+
redis.call('DEL', KEYS[3])
|
|
144
|
+
end
|
|
145
|
+
redis.call('SREM', KEYS[4], ARGV[1])
|
|
146
|
+
return 1
|
|
147
|
+
LUA
|
|
148
|
+
|
|
149
|
+
# Unregisters the processes whose heartbeat key has expired. Returns
|
|
150
|
+
# how many.
|
|
151
|
+
#
|
|
152
|
+
# KEYS: the processes set
|
|
153
|
+
# ARGV: the heartbeat keys' prefix (a process's is it and its identity)
|
|
154
|
+
PRUNE = Redis::Script.new(<<~LUA)
|
|
155
|
+
local pruned = 0
|
|
156
|
+
for _, identity in ipairs(redis.call('SMEMBERS', KEYS[1])) do
|
|
157
|
+
if redis.call('EXISTS', ARGV[1] .. identity) == 0 then
|
|
158
|
+
redis.call('SREM', KEYS[1], identity)
|
|
159
|
+
pruned = pruned + 1
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
return pruned
|
|
163
|
+
LUA
|
|
164
|
+
|
|
165
|
+
# Every queue's size and oldest job: name, size, job, name, size,
|
|
166
|
+
# job... (a job is false for an empty queue).
|
|
167
|
+
#
|
|
168
|
+
# KEYS: the queues set
|
|
169
|
+
# ARGV: queue key prefix ("<prefix>queue:")
|
|
170
|
+
QUEUE_SAMPLES = Redis::Script.new(<<~LUA)
|
|
171
|
+
local samples = {}
|
|
172
|
+
for _, queue in ipairs(redis.call('SMEMBERS', KEYS[1])) do
|
|
173
|
+
samples[#samples + 1] = queue
|
|
174
|
+
samples[#samples + 1] = redis.call('LLEN', ARGV[1] .. queue)
|
|
175
|
+
samples[#samples + 1] = redis.call('LINDEX', ARGV[1] .. queue, -1)
|
|
176
|
+
end
|
|
177
|
+
return samples
|
|
178
|
+
LUA
|
|
179
|
+
|
|
180
|
+
# Moves a job from an in-flight key back onto the front of its queue,
|
|
181
|
+
# if it's still there (a worker may have finished it), then
|
|
182
|
+
# unregisters the key if it's empty.
|
|
183
|
+
#
|
|
184
|
+
# KEYS: in-flight key, queue list, queues set, in-flight set
|
|
185
|
+
# ARGV: raw job ("" for none), queue name
|
|
186
|
+
REQUEUE_ONE = Redis::Script.new(<<~LUA)
|
|
187
|
+
local moved = 0
|
|
188
|
+
if ARGV[1] ~= '' and redis.call('LREM', KEYS[1], 1, ARGV[1]) == 1 then
|
|
189
|
+
redis.call('SADD', KEYS[3], ARGV[2])
|
|
190
|
+
redis.call('RPUSH', KEYS[2], ARGV[1])
|
|
191
|
+
moved = 1
|
|
192
|
+
end
|
|
193
|
+
if redis.call('EXISTS', KEYS[1]) == 0 then redis.call('SREM', KEYS[4], KEYS[1]) end
|
|
194
|
+
return moved
|
|
195
|
+
LUA
|
|
196
|
+
|
|
197
|
+
# Acquires or renews a lease.
|
|
198
|
+
#
|
|
199
|
+
# KEYS: lease key
|
|
200
|
+
# ARGV: owner, ttl (ms)
|
|
201
|
+
LEASE = Redis::Script.new(<<~LUA)
|
|
202
|
+
local current = redis.call('GET', KEYS[1])
|
|
203
|
+
if current == ARGV[1] then
|
|
204
|
+
redis.call('PEXPIRE', KEYS[1], ARGV[2])
|
|
205
|
+
return 1
|
|
206
|
+
elseif not current then
|
|
207
|
+
redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
|
|
208
|
+
return 1
|
|
209
|
+
end
|
|
210
|
+
return 0
|
|
211
|
+
LUA
|
|
212
|
+
|
|
213
|
+
# Deletes a key only if it still holds the given value.
|
|
214
|
+
#
|
|
215
|
+
# KEYS: key
|
|
216
|
+
# ARGV: expected value
|
|
217
|
+
RELEASE = Redis::Script.new(<<~LUA)
|
|
218
|
+
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
219
|
+
return redis.call('DEL', KEYS[1])
|
|
220
|
+
end
|
|
221
|
+
return 0
|
|
222
|
+
LUA
|
|
223
|
+
|
|
224
|
+
# Looks for a job by jid in a slice of a queue, so a lookup doesn't
|
|
225
|
+
# copy whole queues out of Redis. Entries are checked from the tail
|
|
226
|
+
# (next to run) backwards; a cheap substring test comes before decoding.
|
|
227
|
+
#
|
|
228
|
+
# KEYS: queue list
|
|
229
|
+
# ARGV: jid, start index, stop index (negative: counted from the tail)
|
|
230
|
+
# Returns the job's JSON (or false) and the queue's length.
|
|
231
|
+
FIND_JID = Redis::Script.new(<<~LUA)
|
|
232
|
+
local items = redis.call('LRANGE', KEYS[1], ARGV[2], ARGV[3])
|
|
233
|
+
local needle = '"' .. ARGV[1] .. '"'
|
|
234
|
+
for i = #items, 1, -1 do
|
|
235
|
+
local raw = items[i]
|
|
236
|
+
if string.find(raw, needle, 1, true) then
|
|
237
|
+
local ok, job = pcall(cjson.decode, raw)
|
|
238
|
+
if ok and type(job) == 'table' and job['jid'] == ARGV[1] then
|
|
239
|
+
return {raw, redis.call('LLEN', KEYS[1])}
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
return {false, redis.call('LLEN', KEYS[1])}
|
|
244
|
+
LUA
|
|
245
|
+
|
|
246
|
+
# Deletes the jobs of one class from a slice of a queue. The class is
|
|
247
|
+
# the one the dashboard shows: display_class when set (an ActiveJob's
|
|
248
|
+
# own class), else class.
|
|
249
|
+
#
|
|
250
|
+
# KEYS: queue list
|
|
251
|
+
# ARGV: class name, start index, stop index (negative: from the tail)
|
|
252
|
+
# Returns how many jobs it deleted and the queue's length after.
|
|
253
|
+
DELETE_CLASS = Redis::Script.new(<<~LUA)
|
|
254
|
+
local items = redis.call('LRANGE', KEYS[1], ARGV[2], ARGV[3])
|
|
255
|
+
local needle = '"' .. ARGV[1] .. '"'
|
|
256
|
+
local deleted = 0
|
|
257
|
+
for i = #items, 1, -1 do
|
|
258
|
+
local raw = items[i]
|
|
259
|
+
if string.find(raw, needle, 1, true) then
|
|
260
|
+
local ok, job = pcall(cjson.decode, raw)
|
|
261
|
+
if ok and type(job) == 'table' then
|
|
262
|
+
local name = job['display_class']
|
|
263
|
+
if type(name) ~= 'string' then name = job['class'] end
|
|
264
|
+
if name == ARGV[1] then
|
|
265
|
+
deleted = deleted + redis.call('LREM', KEYS[1], -1, raw)
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
return {deleted, redis.call('LLEN', KEYS[1])}
|
|
271
|
+
LUA
|
|
272
|
+
|
|
273
|
+
# The dashboard's counts: processed, failed, the schedule's, retry
|
|
274
|
+
# set's and dead set's sizes, the default queue's oldest job (false
|
|
275
|
+
# when empty), the queues and their sizes, and every live process's
|
|
276
|
+
# busy count.
|
|
277
|
+
#
|
|
278
|
+
# KEYS: queues set, processed and failed counters, schedule, retry and
|
|
279
|
+
# dead sets, processes set
|
|
280
|
+
# ARGV: queue key prefix ("<prefix>queue:"), process key prefix
|
|
281
|
+
STATS = Redis::Script.new(<<~LUA)
|
|
282
|
+
local queues = redis.call('SMEMBERS', KEYS[1])
|
|
283
|
+
local sizes = {}
|
|
284
|
+
for i, queue in ipairs(queues) do sizes[i] = redis.call('LLEN', ARGV[1] .. queue) end
|
|
285
|
+
local busy = {}
|
|
286
|
+
for _, identity in ipairs(redis.call('SMEMBERS', KEYS[7])) do
|
|
287
|
+
-- A process that died recently may still be listed without a hash.
|
|
288
|
+
local count = redis.call('HGET', ARGV[2] .. identity, 'busy')
|
|
289
|
+
if count then busy[#busy + 1] = count end
|
|
290
|
+
end
|
|
291
|
+
return {redis.call('GET', KEYS[2]), redis.call('GET', KEYS[3]), redis.call('ZCARD', KEYS[4]),
|
|
292
|
+
redis.call('ZCARD', KEYS[5]), redis.call('ZCARD', KEYS[6]), redis.call('LINDEX', ARGV[1] .. 'default', -1),
|
|
293
|
+
queues, sizes, busy}
|
|
294
|
+
LUA
|
|
295
|
+
|
|
296
|
+
# The hash of each member of a set: member, its hash's fields and
|
|
297
|
+
# values (as HGETALL gives them), member, ... The hash of member m is
|
|
298
|
+
# at ARGV[1] .. m .. ARGV[2].
|
|
299
|
+
#
|
|
300
|
+
# KEYS: the set
|
|
301
|
+
MEMBER_HASHES = Redis::Script.new(<<~LUA)
|
|
302
|
+
local found = {}
|
|
303
|
+
for _, member in ipairs(redis.call('SMEMBERS', KEYS[1])) do
|
|
304
|
+
found[#found + 1] = member
|
|
305
|
+
found[#found + 1] = redis.call('HGETALL', ARGV[1] .. member .. ARGV[2])
|
|
306
|
+
end
|
|
307
|
+
return found
|
|
308
|
+
LUA
|
|
309
|
+
|
|
310
|
+
# The hash of each member of +set+, in one round trip: [member, hash]
|
|
311
|
+
# pairs, by member. Member m's hash is at +prefix+, m, +suffix+.
|
|
312
|
+
def self.member_hashes(config, set, prefix, suffix = "")
|
|
313
|
+
config.redis { |c| c.evalsha(MEMBER_HASHES, [set], [prefix, suffix]) }
|
|
314
|
+
.each_slice(2).map { |member, row| [member, row.each_slice(2).to_h] }.sort_by(&:first)
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# Raises hash fields to the given values if they are larger.
|
|
318
|
+
#
|
|
319
|
+
# KEYS: hash, ARGV: ttl seconds, field1, value1, field2, value2...
|
|
320
|
+
HMAX = Redis::Script.new(<<~LUA)
|
|
321
|
+
for i = 2, #ARGV, 2 do
|
|
322
|
+
local cur = tonumber(redis.call('HGET', KEYS[1], ARGV[i]) or '-1')
|
|
323
|
+
if tonumber(ARGV[i + 1]) > cur then
|
|
324
|
+
redis.call('HSET', KEYS[1], ARGV[i], ARGV[i + 1])
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
redis.call('EXPIRE', KEYS[1], ARGV[1])
|
|
328
|
+
return 1
|
|
329
|
+
LUA
|
|
330
|
+
end
|
|
331
|
+
end
|
data/lib/robin/statsd.rb
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
|
|
5
|
+
module Robin
|
|
6
|
+
# Sends job and queue metrics to StatsD or DogStatsD over UDP.
|
|
7
|
+
#
|
|
8
|
+
# Robin.configure_server do |config|
|
|
9
|
+
# config.statsd = Robin::Statsd.new(host: "127.0.0.1", port: 8125,
|
|
10
|
+
# prefix: "robin", tags: {env: "production"}, dogstatsd: true)
|
|
11
|
+
# end
|
|
12
|
+
#
|
|
13
|
+
# Emitted (with a "class" or "queue" tag under DogStatsD, or folded into
|
|
14
|
+
# the metric name for plain StatsD):
|
|
15
|
+
# jobs.success, jobs.failure (counters), jobs.perform (timing, ms avg)
|
|
16
|
+
# queue.size, queue.latency (gauges, seconds; sampled by the leader)
|
|
17
|
+
# jobs.recovered (counter: in-flight jobs put back from dead processes)
|
|
18
|
+
# jobs.orphaned (counter: in-flight jobs no worker held, put back)
|
|
19
|
+
# jobs.poison (counter, by class: jobs moved to the dead set for
|
|
20
|
+
# crashing their process too often)
|
|
21
|
+
# batch.created, batch.complete, batch.success, batch.death,
|
|
22
|
+
# batch.cancelled (counters), batch.duration (timing, ms from
|
|
23
|
+
# creation to completion); under DogStatsD, complete, success, death
|
|
24
|
+
# and duration carry the batch's tags ("tag:<tag>" for each)
|
|
25
|
+
#
|
|
26
|
+
# With +distributions: true+ (DogStatsD only), it also sends Datadog
|
|
27
|
+
# distributions, for percentiles across every process:
|
|
28
|
+
# jobs.perform_dist (ms, by class; one packet per successful job)
|
|
29
|
+
# batch.duration_dist (ms)
|
|
30
|
+
#
|
|
31
|
+
# Sending never blocks and never raises into Robin.
|
|
32
|
+
class Statsd
|
|
33
|
+
MAX_PACKET = 1_432
|
|
34
|
+
|
|
35
|
+
attr_reader :host, :port, :prefix, :tags
|
|
36
|
+
|
|
37
|
+
def initialize(host: "127.0.0.1", port: 8125, prefix: "robin", tags: {}, dogstatsd: false, distributions: false)
|
|
38
|
+
raise ArgumentError, "distributions need dogstatsd: true" if distributions && !dogstatsd
|
|
39
|
+
|
|
40
|
+
@host = host
|
|
41
|
+
@port = port
|
|
42
|
+
@prefix = prefix.to_s.empty? ? "" : "#{prefix}."
|
|
43
|
+
@tags = tags
|
|
44
|
+
@dogstatsd = dogstatsd
|
|
45
|
+
@distributions = distributions
|
|
46
|
+
@mutex = Mutex.new
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def dogstatsd? = @dogstatsd
|
|
50
|
+
|
|
51
|
+
def distributions? = @distributions
|
|
52
|
+
|
|
53
|
+
def increment(name, value = 1, tags: {}) = send_lines([line(name, value, "c", tags)])
|
|
54
|
+
|
|
55
|
+
def gauge(name, value, tags: {}) = send_lines([line(name, value, "g", tags)])
|
|
56
|
+
|
|
57
|
+
def timing(name, ms, tags: {}) = send_lines([line(name, ms.round(2), "ms", tags)])
|
|
58
|
+
|
|
59
|
+
# A Datadog distribution sample (needs dogstatsd: true).
|
|
60
|
+
def distribution(name, value, tags: {}) = send_lines([line(name, value.round(2), "d", tags)])
|
|
61
|
+
|
|
62
|
+
# +jobs+: {class => {s:, f:, ms:, hist:}} as drained by Metrics::Collector.
|
|
63
|
+
def job_metrics(jobs)
|
|
64
|
+
lines = jobs.flat_map do |klass, d|
|
|
65
|
+
tags = {class: klass}
|
|
66
|
+
out = [] #: Array[String]
|
|
67
|
+
success = d.fetch(:s)
|
|
68
|
+
failure = d.fetch(:f)
|
|
69
|
+
out << line("jobs.success", success, "c", tags) if success > 0
|
|
70
|
+
out << line("jobs.failure", failure, "c", tags) if failure > 0
|
|
71
|
+
# The collector times successful runs only.
|
|
72
|
+
out << line("jobs.perform", (d.fetch(:ms) / success).round(2), "ms", tags) if success > 0
|
|
73
|
+
out
|
|
74
|
+
end
|
|
75
|
+
send_lines(lines)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# +queues+: {name => {size:, latency:}}
|
|
79
|
+
def queue_metrics(queues)
|
|
80
|
+
lines = queues.flat_map do |name, q|
|
|
81
|
+
tags = {queue: name}
|
|
82
|
+
[line("queue.size", q.fetch(:size), "g", tags), line("queue.latency", q.fetch(:latency), "g", tags)]
|
|
83
|
+
end
|
|
84
|
+
send_lines(lines)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def close
|
|
88
|
+
@mutex.synchronize do
|
|
89
|
+
@socket&.close
|
|
90
|
+
@socket = nil
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
private
|
|
95
|
+
|
|
96
|
+
def line(name, value, type, tags)
|
|
97
|
+
if @dogstatsd
|
|
98
|
+
out = "#{@prefix}#{name}:#{value}|#{type}"
|
|
99
|
+
all = @tags.merge(tags)
|
|
100
|
+
# An Array value repeats the tag once per element.
|
|
101
|
+
pairs = all.flat_map { |k, v| Array(v).map { |each| "#{k}:#{each.to_s.tr("|,#", "___")}" } }
|
|
102
|
+
out << "|#" << pairs.join(",") unless pairs.empty?
|
|
103
|
+
out
|
|
104
|
+
else
|
|
105
|
+
extra = tags.values.map { |v| v.to_s.gsub(/[^A-Za-z0-9_-]/, "_") }
|
|
106
|
+
"#{@prefix}#{[name, *extra].join(".")}:#{value}|#{type}"
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Packs lines into datagrams below the safe UDP payload size.
|
|
111
|
+
def send_lines(lines)
|
|
112
|
+
return if lines.empty?
|
|
113
|
+
|
|
114
|
+
packet = +""
|
|
115
|
+
lines.each do |l|
|
|
116
|
+
if !packet.empty? && packet.bytesize + l.bytesize + 1 > MAX_PACKET
|
|
117
|
+
deliver(packet)
|
|
118
|
+
packet = +""
|
|
119
|
+
end
|
|
120
|
+
packet << "\n" unless packet.empty?
|
|
121
|
+
packet << l
|
|
122
|
+
end
|
|
123
|
+
deliver(packet)
|
|
124
|
+
nil
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def deliver(packet)
|
|
128
|
+
@mutex.synchronize do
|
|
129
|
+
(@socket ||= UDPSocket.new.tap { |s| s.connect(@host, @port) }).write_nonblock(packet)
|
|
130
|
+
end
|
|
131
|
+
rescue IOError, SystemCallError, SocketError
|
|
132
|
+
@socket = nil
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|