robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
data/lib/robin/client.rb
CHANGED
|
@@ -1,37 +1,371 @@
|
|
|
1
|
-
|
|
2
|
-
require_relative 'user'
|
|
3
|
-
require 'twitter'
|
|
1
|
+
# frozen_string_literal: true
|
|
4
2
|
|
|
5
3
|
module Robin
|
|
4
|
+
# Pushes jobs into Redis.
|
|
5
|
+
#
|
|
6
|
+
# Robin::Client.push("class" => "ReportJob", "args" => [1], "queue" => "low")
|
|
7
|
+
# Robin::Client.push_bulk("class" => ReportJob, "args" => [[1], [2], [3]])
|
|
6
8
|
class Client
|
|
7
|
-
|
|
8
|
-
@client = Twitter::Client.new(
|
|
9
|
-
:consumer_key => Robin::Config.key,
|
|
10
|
-
:consumer_secret => Robin::Config.secret,
|
|
11
|
-
:oauth_token => Robin::User.token,
|
|
12
|
-
:oauth_token_secret => Robin::User.secret)
|
|
9
|
+
class ArgumentError < ::ArgumentError; end
|
|
13
10
|
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
class << self
|
|
12
|
+
# A callable that may take over delivery (Robin::Testing uses it to
|
|
13
|
+
# keep jobs in memory). It returns truthy when it handled the push.
|
|
14
|
+
attr_accessor :interceptor
|
|
15
|
+
|
|
16
|
+
# Callables that may rewrite a job hash (string keys) before #push
|
|
17
|
+
# handles it, each returning the hash to use. The Rails integration
|
|
18
|
+
# adds one that turns an ActiveJob class into its wrapper, so
|
|
19
|
+
# periodic jobs (and Client.push) can name ActiveJob classes.
|
|
20
|
+
def rewriters = (@rewriters ||= [])
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Sends every push made by this thread inside the block to +pool+ (a
|
|
24
|
+
# Robin::Redis::Pool) instead of the configured Redis, e.g. to spread
|
|
25
|
+
# jobs over several Redis shards:
|
|
26
|
+
#
|
|
27
|
+
# SHARDS = [Robin::Redis::Pool.new { Robin::Redis::Connection.new(url: A) }, ...]
|
|
28
|
+
# Robin::Client.via(SHARDS.sample) { ReportJob.perform_async(1) }
|
|
29
|
+
#
|
|
30
|
+
# Each shard needs its own Robin processes to work its jobs.
|
|
31
|
+
def self.via(pool)
|
|
32
|
+
previous = Thread.current[:robin_client_pool]
|
|
33
|
+
Thread.current[:robin_client_pool] = pool
|
|
34
|
+
yield
|
|
35
|
+
ensure
|
|
36
|
+
Thread.current[:robin_client_pool] = previous
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# The pool set by an enclosing Client.via block, if any.
|
|
40
|
+
def self.current_pool = Thread.current[:robin_client_pool]
|
|
41
|
+
|
|
42
|
+
# Holds back the jobs this thread's pushes send inside the block, and
|
|
43
|
+
# sends them all when it finishes: in one round trip when they go
|
|
44
|
+
# straight to the same Redis. If the block raises, none are sent.
|
|
45
|
+
# Returns the block's value.
|
|
46
|
+
#
|
|
47
|
+
# Robin::Client.together { reports.each { |r| ReportJob.perform_async(r.id) } }
|
|
48
|
+
#
|
|
49
|
+
# mutant:disable -- on CRuby, setting a fiber-local to nil removes it,
|
|
50
|
+
# so key? would do; on JRuby the key stays, and every later block would
|
|
51
|
+
# take itself for a nested one.
|
|
52
|
+
def self.together
|
|
53
|
+
return yield if Thread.current[:robin_client_together]
|
|
54
|
+
|
|
55
|
+
held = Thread.current[:robin_client_together] = []
|
|
56
|
+
begin
|
|
57
|
+
result = yield
|
|
58
|
+
ensure
|
|
59
|
+
Thread.current[:robin_client_together] = nil
|
|
60
|
+
end
|
|
61
|
+
send_together(held)
|
|
62
|
+
result
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Sends what a Client.together block held: [client, payloads] each.
|
|
66
|
+
def self.send_together(held)
|
|
67
|
+
return if held.empty?
|
|
68
|
+
|
|
69
|
+
pool = held.first.first.redis_pool
|
|
70
|
+
if held.all? { |client, _payloads| client.redis_pool.equal?(pool) && client.plain? }
|
|
71
|
+
pool.with { |conn| conn.pipelined { |pipe| held.each { |client, payloads| client.queue_push(pipe, payloads) } } }
|
|
72
|
+
else
|
|
73
|
+
held.each { |client, payloads| client.__send__(:send_payloads, payloads) }
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
private_class_method :send_together
|
|
77
|
+
|
|
78
|
+
def self.push(item) = new.push(item)
|
|
79
|
+
|
|
80
|
+
def self.push_bulk(items) = new.push_bulk(items)
|
|
81
|
+
|
|
82
|
+
# Converts a delay (seconds), a Time, or an epoch timestamp (seconds or
|
|
83
|
+
# milliseconds) to epoch milliseconds.
|
|
84
|
+
def self.timestamp_ms(value, now = Robin.now_ms)
|
|
85
|
+
case value
|
|
86
|
+
when nil then raise ArgumentError, "a time is required, not nil"
|
|
87
|
+
when Numeric
|
|
88
|
+
if value > 100_000_000_000 then Integer(value) # already epoch milliseconds
|
|
89
|
+
elsif value > 1_000_000_000 then Integer(value * 1000) # epoch seconds
|
|
90
|
+
else now + Integer(value * 1000) # a delay in seconds
|
|
91
|
+
end
|
|
92
|
+
when Time then Integer(value.to_r * 1000) # Time#to_time needs the date library
|
|
93
|
+
else
|
|
94
|
+
if value.respond_to?(:to_time) && !value.respond_to?(:parts)
|
|
95
|
+
Integer(value.to_time.to_r * 1000) # Date, DateTime...
|
|
96
|
+
else
|
|
97
|
+
now + Integer(value.to_f * 1000) # e.g. ActiveSupport::Duration
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# +count+ times spread evenly over +interval+ seconds, the first at
|
|
103
|
+
# +start+ (anything #timestamp_ms takes; default now), as epoch
|
|
104
|
+
# milliseconds.
|
|
105
|
+
def self.spread(count, interval, start = 0)
|
|
106
|
+
raise ArgumentError, "'spread_interval' must be a positive number of seconds" unless interval.is_a?(Numeric) && interval.positive?
|
|
107
|
+
|
|
108
|
+
first = timestamp_ms(start)
|
|
109
|
+
Array.new(count) { |i| first + Integer(interval * 1000 * i / count) }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
attr_reader :config
|
|
113
|
+
|
|
114
|
+
# +pool+ pushes to another Redis (see Client.via); +chain+ replaces the
|
|
115
|
+
# configured client middleware for this client's pushes.
|
|
116
|
+
def initialize(config: Robin.config, pool: nil, chain: nil)
|
|
117
|
+
@config = config
|
|
118
|
+
@pool = pool || Client.current_pool
|
|
119
|
+
@chain = chain
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def middleware = @chain || @config.client_middleware
|
|
123
|
+
|
|
124
|
+
def redis_pool
|
|
125
|
+
@pool || @config.redis_pool
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Pushes a single job. Returns its jid, or nil if a client middleware
|
|
129
|
+
# stopped the push (e.g. a duplicate unique job).
|
|
130
|
+
def push(item)
|
|
131
|
+
item = Client.rewriters.reduce(item.transform_keys(&:to_s)) { |rewritten, rewriter| rewriter.call(rewritten) }
|
|
132
|
+
normalized = normalize(item)
|
|
133
|
+
# Inside a batch's +jobs+ block, the job joins the batch as it's sent
|
|
134
|
+
# (see Batch.holding).
|
|
135
|
+
payload = Batch.holding do
|
|
136
|
+
pushed = invoke_middleware(normalized)
|
|
137
|
+
deliver([pushed]) if pushed
|
|
138
|
+
pushed
|
|
139
|
+
end
|
|
140
|
+
payload&.fetch("jid")
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Pushes many jobs of the same class at once. "args" is an array of
|
|
144
|
+
# argument arrays; "at" may be a single time or one per job, and
|
|
145
|
+
# "spread_interval" (seconds) spreads the jobs evenly over that long,
|
|
146
|
+
# from "at" or now. Returns the jids (nil for any job stopped by
|
|
147
|
+
# middleware).
|
|
148
|
+
def push_bulk(item)
|
|
149
|
+
base = item.transform_keys(&:to_s)
|
|
150
|
+
args_list = base["args"]
|
|
151
|
+
raise ArgumentError, "push_bulk needs 'args' to be an Array of Arrays" unless args_list.is_a?(Array) && args_list.all?(Array)
|
|
152
|
+
|
|
153
|
+
ats = base["at"]
|
|
154
|
+
if ats.is_a?(Array) && !ats.size.equal?(args_list.size)
|
|
155
|
+
raise ArgumentError, "push_bulk 'at' must have the same size as 'args'"
|
|
156
|
+
end
|
|
157
|
+
if (interval = base.delete("spread_interval"))
|
|
158
|
+
raise ArgumentError, "push_bulk takes 'spread_interval' or an 'at' per job, not both" if ats.is_a?(Array)
|
|
159
|
+
|
|
160
|
+
# (Steep can't tell that ats, not an Array, splats to one argument.)
|
|
161
|
+
ats = Client.spread(args_list.size, interval, *ats) # steep:ignore UnexpectedPositionalArgument
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
jids = base.delete("jids")
|
|
165
|
+
jid = base.delete("jid")
|
|
166
|
+
# What every job shares is worked out once; then each gets its own
|
|
167
|
+
# arguments, jid and (when given one per job) time.
|
|
168
|
+
template = normalize(base.merge({"args" => [], "at" => (ats unless ats.is_a?(Array)), "jid" => ""}))
|
|
169
|
+
built = args_list.each_with_index.map { |args, i| build(template, args, jids&.fetch(i) || jid, (ats.fetch(i) if ats.is_a?(Array))) }
|
|
170
|
+
# Inside a batch's +jobs+ block, the jobs join the batch together.
|
|
171
|
+
payloads = Batch.holding do
|
|
172
|
+
# Their unique locks are taken together too.
|
|
173
|
+
Unique.claiming(middleware, built) { built.map { |payload| run_middleware(payload) } }
|
|
174
|
+
.tap { |list| deliver(list.compact) }
|
|
175
|
+
end
|
|
176
|
+
payloads.map { |p| p&.fetch("jid") }
|
|
16
177
|
end
|
|
17
178
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
179
|
+
# Fills in defaults and validates a job hash. Mutates and returns it.
|
|
180
|
+
def normalize(item)
|
|
181
|
+
item = item.transform_keys(&:to_s)
|
|
182
|
+
klass = item["class"]
|
|
183
|
+
raise ArgumentError, "job must have a 'class'" unless klass
|
|
184
|
+
raise ArgumentError, "job 'args' must be an Array" unless item["args"].is_a?(Array)
|
|
185
|
+
|
|
186
|
+
if klass.instance_of?(Class)
|
|
187
|
+
# (A job class: Steep only knows it's a Class.)
|
|
188
|
+
item = klass.robin_options.merge(item) if klass.respond_to?(:robin_options) # steep:ignore NoMethod
|
|
189
|
+
item["class"] = klass.name
|
|
21
190
|
else
|
|
22
|
-
|
|
191
|
+
item["class"] = klass.to_s
|
|
23
192
|
end
|
|
193
|
+
queue = item["queue"].to_s
|
|
194
|
+
item["queue"] = queue.empty? ? "default" : queue
|
|
195
|
+
item["retry"] = true unless item.key?("retry")
|
|
196
|
+
item["jid"] ||= Robin.generate_jid
|
|
197
|
+
item["created_at"] ||= Robin.now_ms
|
|
198
|
+
at = item.delete("at")
|
|
199
|
+
at_ms = self.class.timestamp_ms(at) if at
|
|
200
|
+
item["at"] = Robin.epoch_seconds(at_ms) if at_ms
|
|
201
|
+
if (expires_in = item.delete("expires_in"))
|
|
202
|
+
# The clock starts when the job is due to run.
|
|
203
|
+
item["expires_at"] = [at_ms, item.fetch("created_at")].compact.max + Integer(expires_in.to_f * 1000)
|
|
204
|
+
end
|
|
205
|
+
if (window = item["retry_for"]) && !(window.is_a?(Numeric) && window.positive?)
|
|
206
|
+
raise ArgumentError, "job 'retry_for' must be a positive number of seconds"
|
|
207
|
+
end
|
|
208
|
+
item["retry_queue"] = item.fetch("retry_queue").to_s if item["retry_queue"]
|
|
209
|
+
%w[wait wait_until sync].each { |k| item.delete(k) }
|
|
210
|
+
check_args(item) if @config.strict_args
|
|
211
|
+
item
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
private
|
|
215
|
+
|
|
216
|
+
# The work push_bulk does for each job is in methods of their own:
|
|
217
|
+
# JRuby compiles a method after 50 calls, so these are compiled after
|
|
218
|
+
# 50 jobs, rather than with push_bulk after 50 bulk pushes.
|
|
219
|
+
|
|
220
|
+
# A bulk-pushed job: +template+ with its own arguments, jid (or a new
|
|
221
|
+
# one) and time, if it has one.
|
|
222
|
+
def build(template, args, jid, at)
|
|
223
|
+
payload = template.merge({"args" => args, "jid" => jid || Robin.generate_jid})
|
|
224
|
+
schedule(payload, self.class.timestamp_ms(at)) if at
|
|
225
|
+
payload
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# The job as client middleware leaves it, or nil if it stopped it.
|
|
229
|
+
def run_middleware(payload)
|
|
230
|
+
check_args(payload) if @config.strict_args
|
|
231
|
+
invoke_middleware(payload)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def invoke_middleware(payload)
|
|
235
|
+
middleware.invoke(payload.fetch("class"), payload, payload.fetch("queue"), redis_pool) { payload }
|
|
236
|
+
.tap { |result| Debounce.carry(payload, result) }
|
|
24
237
|
end
|
|
25
238
|
|
|
26
|
-
|
|
27
|
-
|
|
239
|
+
# Sets a bulk-pushed job's own time (epoch ms); its expiry, if any,
|
|
240
|
+
# counts from then (the template's counted from its creation).
|
|
241
|
+
def schedule(payload, at_ms)
|
|
242
|
+
payload["at"] = Robin.epoch_seconds(at_ms)
|
|
243
|
+
payload["expires_at"] += [at_ms - payload.fetch("created_at"), 0].max if payload["expires_at"]
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Config#push_deferrer may take over delivery (e.g. until a database
|
|
247
|
+
# transaction commits); otherwise the jobs go out now.
|
|
248
|
+
def deliver(payloads)
|
|
249
|
+
# An atomic batch block sends its jobs once it finishes.
|
|
250
|
+
hold = Batch.hold
|
|
251
|
+
return hold.pushes << [self, payloads] if hold
|
|
252
|
+
|
|
253
|
+
deliver_now(payloads)
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
public
|
|
257
|
+
|
|
258
|
+
# Would #deliver_now send these jobs to Redis right away (with no
|
|
259
|
+
# interceptor, push deferrer or reliable push buffer in the way)?
|
|
260
|
+
def direct?(payloads)
|
|
261
|
+
plain? && (!@config.push_deferrer || payloads.any? { |p| p.key?("bid") })
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# Does #send_payloads go straight to Redis (with no interceptor or
|
|
265
|
+
# reliable push buffer in the way)?
|
|
266
|
+
def plain? = !Client.interceptor && !@config.reliable_push
|
|
267
|
+
|
|
268
|
+
# Delivers now, without waiting for an atomic batch block (see
|
|
269
|
+
# Batch#jobs).
|
|
270
|
+
def deliver_now(payloads)
|
|
271
|
+
# Batch jobs are never deferred: their batch has already counted them,
|
|
272
|
+
# and a deferred push that never happens (e.g. a rolled-back
|
|
273
|
+
# transaction) would leave the batch waiting forever.
|
|
274
|
+
deferrable = payloads.none? { |p| p.key?("bid") }
|
|
275
|
+
return if deferrable && @config.push_deferrer&.call(payloads) { send_payloads(payloads) }
|
|
276
|
+
|
|
277
|
+
send_payloads(payloads)
|
|
28
278
|
end
|
|
29
279
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
280
|
+
private
|
|
281
|
+
|
|
282
|
+
def send_payloads(payloads)
|
|
283
|
+
held = Thread.current[:robin_client_together]
|
|
284
|
+
return held << [self, payloads] if held
|
|
285
|
+
|
|
286
|
+
if @config.reliable_push
|
|
287
|
+
ReliablePush.for(@config, @pool).push(payloads) { raw_push(payloads) }
|
|
33
288
|
else
|
|
34
|
-
|
|
289
|
+
raw_push(payloads)
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
def raw_push(payloads)
|
|
294
|
+
return if Client.interceptor&.call(payloads)
|
|
295
|
+
|
|
296
|
+
redis_pool.with { |conn| conn.pipelined { |pipe| queue_push(pipe, payloads) } }
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
public
|
|
300
|
+
|
|
301
|
+
# Queues the commands pushing +payloads+ into +pipe+ (a pipeline or
|
|
302
|
+
# transaction on #redis_pool).
|
|
303
|
+
def queue_push(pipe, payloads)
|
|
304
|
+
Debounce.queue_claims(pipe, payloads)
|
|
305
|
+
now = Robin.now_ms
|
|
306
|
+
due = Robin.epoch_seconds(now)
|
|
307
|
+
payloads.group_by { |p| (p.key?("at") && p.fetch("at") > due) ? :later : p.fetch("queue") }.each do |queue, group|
|
|
308
|
+
if queue == :later
|
|
309
|
+
# The score says when; enqueued_at is set when it's moved onto
|
|
310
|
+
# its queue.
|
|
311
|
+
args = group.flat_map { |p| [p.fetch("at"), Robin.dump_json(p.except("at", "enqueued_at"))] }
|
|
312
|
+
pipe.call("ZADD", @config.key("schedule"), *args)
|
|
313
|
+
else
|
|
314
|
+
entries = group.map do |p|
|
|
315
|
+
p.delete("at")
|
|
316
|
+
p["enqueued_at"] = now
|
|
317
|
+
Robin.dump_json(p)
|
|
318
|
+
end
|
|
319
|
+
pipe.call("SADD", @config.key("queues"), queue)
|
|
320
|
+
pipe.call("LPUSH", @config.key("queue", queue), *entries)
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
index_jids(pipe, payloads)
|
|
324
|
+
end
|
|
325
|
+
|
|
326
|
+
private
|
|
327
|
+
|
|
328
|
+
# Records each job's queue for Robin::JobLocator (see Config#jid_index).
|
|
329
|
+
def index_jids(pipe, payloads)
|
|
330
|
+
ttl = @config.jid_index
|
|
331
|
+
return unless ttl
|
|
332
|
+
|
|
333
|
+
payloads.each { |p| pipe.call("SET", @config.key("jid", p.fetch("jid")), p.fetch("queue"), "EX", ttl) }
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
def check_args(item)
|
|
337
|
+
bad = find_non_native(item.fetch("args"))
|
|
338
|
+
return unless bad
|
|
339
|
+
|
|
340
|
+
message = "Job arguments for #{item["class"]} must be JSON-native types (String, Integer, finite Float, " \
|
|
341
|
+
"true, false, nil, Array, or Hash with String keys); got #{bad.inspect} (#{bad.class}). " \
|
|
342
|
+
"Pass IDs or plain values instead of objects."
|
|
343
|
+
case @config.strict_args
|
|
344
|
+
when :warn then @config.logger.warn(message)
|
|
345
|
+
else raise ArgumentError, message
|
|
346
|
+
end
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
# Returns the first value (or hash key) that doesn't survive a JSON round
|
|
350
|
+
# trip unchanged, or nil. (Such a value is never nil or false: those are
|
|
351
|
+
# JSON-native, and come back from the last branch as "nothing found".)
|
|
352
|
+
def find_non_native(obj)
|
|
353
|
+
case obj
|
|
354
|
+
when String, Integer, true then nil
|
|
355
|
+
when Float then (obj unless obj.finite?) # (JSON has no NaN or Infinity)
|
|
356
|
+
when Array
|
|
357
|
+
obj.each do |o|
|
|
358
|
+
bad = find_non_native(o)
|
|
359
|
+
return bad if bad
|
|
360
|
+
end
|
|
361
|
+
nil
|
|
362
|
+
when Hash
|
|
363
|
+
obj.each do |k, v|
|
|
364
|
+
bad = k.is_a?(String) ? find_non_native(v) : k
|
|
365
|
+
return bad if bad
|
|
366
|
+
end
|
|
367
|
+
nil
|
|
368
|
+
else obj
|
|
35
369
|
end
|
|
36
370
|
end
|
|
37
371
|
end
|