robin 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/robin_resp/extconf.rb +29 -0
- data/ext/robin_resp/robin_resp.c +645 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +355 -21
- data/lib/robin/config.rb +454 -4
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +258 -8
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +274 -44
- data/bin/robin +0 -5
- data/lib/robin/actions/authenticate.rb +0 -38
- data/lib/robin/actions/empty_tweet.rb +0 -7
- data/lib/robin/actions/followers.rb +0 -22
- data/lib/robin/actions/help.rb +0 -16
- data/lib/robin/actions/home_timeline.rb +0 -9
- data/lib/robin/actions/new_tweet.rb +0 -11
- data/lib/robin/actions/unknown_flag.rb +0 -11
- data/lib/robin/actions/user_timeline.rb +0 -19
- data/lib/robin/args_parser.rb +0 -70
- data/lib/robin/pin_authenticator.rb +0 -31
- data/lib/robin/user.rb +0 -36
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Reliable fetch: jobs move atomically from their queue into an in-flight
|
|
3
|
+
# key of their process, and leave it once they finish.
|
|
4
|
+
class Fetcher
|
|
5
|
+
# A fetched job.
|
|
6
|
+
class UnitOfWork
|
|
7
|
+
# The queue the job came from.
|
|
8
|
+
attr_reader queue: String
|
|
9
|
+
|
|
10
|
+
# The job's JSON, as fetched.
|
|
11
|
+
attr_reader raw: String
|
|
12
|
+
|
|
13
|
+
attr_reader fetcher: Fetcher
|
|
14
|
+
|
|
15
|
+
# The job's in-flight key.
|
|
16
|
+
attr_reader key: String
|
|
17
|
+
|
|
18
|
+
def initialize: (String queue, String raw, Fetcher fetcher, String key) -> void
|
|
19
|
+
|
|
20
|
+
# Removes the job from its in-flight key, right away or as part of
|
|
21
|
+
# +conn+ (a pipeline or transaction).
|
|
22
|
+
def acknowledge: (?Redis::_Caller? conn) -> void
|
|
23
|
+
|
|
24
|
+
# Puts the job back on the front of its queue.
|
|
25
|
+
def requeue: () -> nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
STALE: Integer
|
|
29
|
+
|
|
30
|
+
ACK_WAIT: Float
|
|
31
|
+
|
|
32
|
+
LARGE: Integer
|
|
33
|
+
|
|
34
|
+
self.@minted: Integer
|
|
35
|
+
self.@minting: Thread::Mutex
|
|
36
|
+
|
|
37
|
+
# A number for a new in-flight key, unique within the process.
|
|
38
|
+
def self.mint: () -> Integer
|
|
39
|
+
|
|
40
|
+
@paused_key: String
|
|
41
|
+
@lock: Thread::Mutex
|
|
42
|
+
@buffer: Array[[UnitOfWork, Float]]
|
|
43
|
+
@acks: Array[String]
|
|
44
|
+
@sending: Hash[Thread, Array[String]]
|
|
45
|
+
@free: Array[String]
|
|
46
|
+
@filling: Hash[Thread, Array[String]]
|
|
47
|
+
@lost: Hash[String, true]
|
|
48
|
+
@large: bool
|
|
49
|
+
@wanting: Integer
|
|
50
|
+
@idle: Hash[Thread, true]
|
|
51
|
+
@ack_queued: bool?
|
|
52
|
+
|
|
53
|
+
attr_reader config: Config
|
|
54
|
+
attr_reader capsule: Capsule
|
|
55
|
+
attr_reader inflight_key: String
|
|
56
|
+
|
|
57
|
+
# The paused queues, as the last heartbeat read them.
|
|
58
|
+
attr_accessor paused: Array[String]
|
|
59
|
+
|
|
60
|
+
# A queue the fetcher puts itself on when acknowledgements are left
|
|
61
|
+
# waiting (the launcher's, which sends them).
|
|
62
|
+
attr_accessor acker: Thread::Queue[Fetcher?]
|
|
63
|
+
|
|
64
|
+
def initialize: (Capsule capsule, String identity) -> void
|
|
65
|
+
|
|
66
|
+
# The next job, or nil after waiting briefly with no work available.
|
|
67
|
+
# +ack+ is the in-flight key of the worker's last finished job.
|
|
68
|
+
def fetch: (?Integer slot, ?ack: String?) -> UnitOfWork?
|
|
69
|
+
|
|
70
|
+
# Takes (for the heartbeat to hand back) the jobs fetched for other
|
|
71
|
+
# workers that are from a paused queue or have waited too long.
|
|
72
|
+
def stale: (Array[String] paused) -> Array[UnitOfWork]
|
|
73
|
+
|
|
74
|
+
# Frees the keys of jobs handed back.
|
|
75
|
+
def freed: (Array[UnitOfWork] units) -> void
|
|
76
|
+
|
|
77
|
+
# Acknowledges the finished jobs waiting for a round trip, if any.
|
|
78
|
+
def send_acks: () -> void
|
|
79
|
+
|
|
80
|
+
# In-flight keys this fetcher is responsible for.
|
|
81
|
+
def held: () -> Array[String]
|
|
82
|
+
|
|
83
|
+
# Lets go of +key+ (found unheld on two sweeps); false if it's held now.
|
|
84
|
+
def forget: (String key) -> bool
|
|
85
|
+
|
|
86
|
+
# Queues on +conn+ the removal of the job in +key+.
|
|
87
|
+
def retire: (Redis::_Caller conn, String key) -> void
|
|
88
|
+
|
|
89
|
+
# Puts jobs back on the front of their queues and frees their keys.
|
|
90
|
+
def put_back: (Redis::Connection conn, Array[UnitOfWork] units) -> nil
|
|
91
|
+
|
|
92
|
+
# Puts the jobs fetched but not handed out back on their queues, and
|
|
93
|
+
# acknowledges the finished ones.
|
|
94
|
+
def release: () -> void
|
|
95
|
+
|
|
96
|
+
# Queues on +tx+ the moves of +units+ back onto their queues.
|
|
97
|
+
def hand_back: (Redis::Pipeline tx, Array[UnitOfWork] units) -> void
|
|
98
|
+
|
|
99
|
+
# Acknowledges the finished jobs in +keys+, then frees the keys.
|
|
100
|
+
def acknowledge: (Redis::Connection conn, Array[String] keys) -> void
|
|
101
|
+
|
|
102
|
+
private
|
|
103
|
+
|
|
104
|
+
def wake_acker: () -> void
|
|
105
|
+
|
|
106
|
+
def take_acks: () -> Array[String]
|
|
107
|
+
|
|
108
|
+
def keep: (Array[UnitOfWork] units) -> void
|
|
109
|
+
|
|
110
|
+
def reserve: (Integer count) -> [Array[String], Array[String]]
|
|
111
|
+
|
|
112
|
+
def settle: (Array[String]? unused) -> void
|
|
113
|
+
|
|
114
|
+
def free: (Array[String] keys) -> void
|
|
115
|
+
|
|
116
|
+
def free_keys: (Array[String] keys) -> void
|
|
117
|
+
|
|
118
|
+
def holding: () -> Array[String]
|
|
119
|
+
|
|
120
|
+
def take: (Array[String] order, Array[String] acks, Array[String] keys, Array[String] minted) -> [Array[UnitOfWork], Array[String]]
|
|
121
|
+
|
|
122
|
+
def take_from: (Redis::Connection conn, String queue, Array[String] acks, Array[String] keys, Array[String] minted) -> [Array[UnitOfWork], Array[String]]
|
|
123
|
+
|
|
124
|
+
def register: (Redis::Pipeline pipe, Array[String] minted) -> void
|
|
125
|
+
|
|
126
|
+
def received: (Redis::Connection conn, String queue, Redis::reply paused, Array[String?] raws, Array[String] keys) -> [Array[UnitOfWork], Array[String]]
|
|
127
|
+
|
|
128
|
+
def wait: (Integer slot, Array[String] order) -> UnitOfWork?
|
|
129
|
+
|
|
130
|
+
def send_back: (Redis::Connection conn, Array[UnitOfWork] units) -> void
|
|
131
|
+
|
|
132
|
+
def monotonic: () -> Float
|
|
133
|
+
end
|
|
134
|
+
end
|
data/sig/robin/gvl.rbs
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Measures GVL contention (with the C extension's Robin::NativeGVL).
|
|
3
|
+
class GVL
|
|
4
|
+
# What GVL needs of Robin::NativeGVL.
|
|
5
|
+
interface _Native
|
|
6
|
+
def start: () -> void
|
|
7
|
+
|
|
8
|
+
def stop: () -> void
|
|
9
|
+
|
|
10
|
+
def wait_ns: () -> Integer
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Robin::NativeGVL, where the C extension defines it.
|
|
14
|
+
def self.native: () -> (Object & _Native)?
|
|
15
|
+
|
|
16
|
+
@native: (Object & _Native)?
|
|
17
|
+
@threads: ^() -> Integer
|
|
18
|
+
# nil until #start.
|
|
19
|
+
@last: [Integer, Integer]
|
|
20
|
+
|
|
21
|
+
def initialize: (?(Object & _Native)? native, ?threads: ^() -> Integer) -> void
|
|
22
|
+
|
|
23
|
+
def available?: () -> bool
|
|
24
|
+
|
|
25
|
+
# Starts measuring; #sample then reports from here.
|
|
26
|
+
def start: () -> void
|
|
27
|
+
|
|
28
|
+
def stop: () -> void
|
|
29
|
+
|
|
30
|
+
# The contention since the last sample (or #start), as a percentage, or
|
|
31
|
+
# nil when it isn't measuring.
|
|
32
|
+
def sample: () -> Float?
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
# Nanoseconds waited so far, and now (monotonic nanoseconds).
|
|
37
|
+
def reading: () -> [Integer, Integer]
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# A tiny HTTP endpoint a server opens for liveness and readiness probes.
|
|
3
|
+
class HealthServer
|
|
4
|
+
REASONS: Hash[Integer, String]
|
|
5
|
+
|
|
6
|
+
READ_TIMEOUT: Integer
|
|
7
|
+
|
|
8
|
+
REQUEST_LIMIT: Integer
|
|
9
|
+
|
|
10
|
+
@launcher: Launcher
|
|
11
|
+
@server: TCPServer
|
|
12
|
+
@thread: Thread
|
|
13
|
+
|
|
14
|
+
# The port listened on (assigned by the OS when given 0).
|
|
15
|
+
attr_reader port: Integer
|
|
16
|
+
|
|
17
|
+
def initialize: (Launcher launcher, port: Integer, ?bind: String) -> void
|
|
18
|
+
|
|
19
|
+
def stop: () -> void
|
|
20
|
+
|
|
21
|
+
# The HTTP response for a Launcher#health report.
|
|
22
|
+
def self.response: (Hash[String, untyped] health) -> String
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def serve: () -> nil
|
|
27
|
+
|
|
28
|
+
def answer: (TCPSocket client) -> void
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Long-running jobs that survive deploys and restarts: define
|
|
3
|
+
# +build_enumerator+ and +each_iteration+ instead of +perform+.
|
|
4
|
+
module IterableJob : Job, IterableJob::_Iteration
|
|
5
|
+
# What an iterable job class defines.
|
|
6
|
+
interface _Iteration
|
|
7
|
+
# An Enumerator of [item, cursor] pairs, starting after +cursor+ (nil
|
|
8
|
+
# on the first run); nil when there's nothing to do.
|
|
9
|
+
def build_enumerator: (*untyped args, cursor: untyped) -> Enumerator[[untyped, untyped], untyped]?
|
|
10
|
+
|
|
11
|
+
# Works on one item; +args+ are the job's arguments.
|
|
12
|
+
def each_iteration: (untyped item, *untyped args) -> void
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
CHECKPOINT_SECONDS: Integer
|
|
16
|
+
|
|
17
|
+
STATE_TTL: Integer
|
|
18
|
+
|
|
19
|
+
def self.included: (Module base) -> void
|
|
20
|
+
|
|
21
|
+
module ClassMethods : Module
|
|
22
|
+
# Rejects a +perform+ of its own.
|
|
23
|
+
def method_added: (Symbol name) -> void
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
@cursor: untyped
|
|
27
|
+
@iterations: Integer
|
|
28
|
+
@current_object: untyped
|
|
29
|
+
|
|
30
|
+
# The last finished item's cursor (any JSON-native value).
|
|
31
|
+
attr_reader cursor: untyped
|
|
32
|
+
|
|
33
|
+
# How many items are finished.
|
|
34
|
+
attr_reader iterations: Integer
|
|
35
|
+
|
|
36
|
+
# The item being (or last) worked on.
|
|
37
|
+
attr_reader current_object: untyped
|
|
38
|
+
|
|
39
|
+
# An iterable job's class is a job class.
|
|
40
|
+
def class: () -> Job::ClassMethods
|
|
41
|
+
|
|
42
|
+
# The job as fetched: IterableJob only reads it while it runs, once
|
|
43
|
+
# it's set.
|
|
44
|
+
def payload: () -> job_hash
|
|
45
|
+
|
|
46
|
+
def perform: (*untyped args) -> void
|
|
47
|
+
|
|
48
|
+
# Runs each iteration (the block): override to wrap them.
|
|
49
|
+
def around_iteration: () { () -> void } -> void
|
|
50
|
+
|
|
51
|
+
def on_start: () -> void
|
|
52
|
+
|
|
53
|
+
def on_resume: () -> void
|
|
54
|
+
|
|
55
|
+
def on_cancel: () -> void
|
|
56
|
+
|
|
57
|
+
def on_stop: () -> void
|
|
58
|
+
|
|
59
|
+
def on_complete: () -> void
|
|
60
|
+
|
|
61
|
+
# Iterates an array; the cursor is the index.
|
|
62
|
+
def array_enumerator: [T] (Array[T] array, cursor: Integer?) -> Enumerator[[T, Integer], void]
|
|
63
|
+
|
|
64
|
+
# Iterates Active Record records in primary-key order; the cursor is the
|
|
65
|
+
# last finished record's id.
|
|
66
|
+
def active_record_records_enumerator: (untyped relation, cursor: untyped, ?batch_size: Integer) -> Enumerator[[untyped, untyped], void]
|
|
67
|
+
|
|
68
|
+
# Like active_record_records_enumerator, but yields whole batches.
|
|
69
|
+
def active_record_batches_enumerator: (untyped relation, cursor: untyped, ?batch_size: Integer) -> Enumerator[[Array[untyped], untyped], void]
|
|
70
|
+
|
|
71
|
+
# Iterates a CSV (or any enumerable of rows); the cursor is the row index.
|
|
72
|
+
def csv_enumerator: [T] (Enumerable[T] csv, cursor: Integer?) -> Enumerator[[T, Integer], void]
|
|
73
|
+
|
|
74
|
+
private
|
|
75
|
+
|
|
76
|
+
def active_record_batches: (untyped relation, untyped cursor, Integer batch_size) { (Array[untyped] records) -> void } -> void
|
|
77
|
+
|
|
78
|
+
def safe_checkpoint: (Config config, String key, Integer? total) -> void
|
|
79
|
+
|
|
80
|
+
def warn_slow: (Float seconds, Config config) -> void
|
|
81
|
+
|
|
82
|
+
def checkpoint: (Config config, String key, Integer? total, ?check_cancel: bool) ?{ (Redis::Pipeline tx) -> void } -> bool
|
|
83
|
+
|
|
84
|
+
# [percent, "done/total"], or [nil, nil, {"iterations" => n}].
|
|
85
|
+
def progress_args: (Integer? total) -> [Integer | Float | nil, String?, Hash[String, Integer]]
|
|
86
|
+
|
|
87
|
+
def percent: (Integer total) -> (Integer | Float)
|
|
88
|
+
|
|
89
|
+
def progress_command: (Config config, Integer? total) -> Redis::command
|
|
90
|
+
|
|
91
|
+
def report_progress: (Integer? total) -> void
|
|
92
|
+
|
|
93
|
+
def requeue: (Config config, String key, Integer? total, String reason) -> void
|
|
94
|
+
|
|
95
|
+
def finish: (Config config, String key, Integer? total) -> void
|
|
96
|
+
|
|
97
|
+
def monotonic: () -> Float
|
|
98
|
+
end
|
|
99
|
+
end
|
data/sig/robin/job.rbs
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# A time to run a job at: a delay in seconds, epoch seconds or
|
|
3
|
+
# milliseconds, a Time, or anything with #to_time (a Date) or #to_f (an
|
|
4
|
+
# ActiveSupport::Duration).
|
|
5
|
+
type time = Numeric | Time | _ToTime | _ToF
|
|
6
|
+
|
|
7
|
+
interface _ToTime
|
|
8
|
+
def to_time: () -> Time
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
interface _ToF
|
|
12
|
+
def to_f: () -> Float
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Include in a class to make it a Robin job:
|
|
16
|
+
#
|
|
17
|
+
# class Thumbnail
|
|
18
|
+
# include Robin::Job
|
|
19
|
+
# extend Robin::Job::ClassMethods # in your RBS: include adds it
|
|
20
|
+
# robin_options queue: "media", retry: 10
|
|
21
|
+
#
|
|
22
|
+
# def perform(image_id, size) = ...
|
|
23
|
+
# end
|
|
24
|
+
module Job
|
|
25
|
+
# Raised inside a job that runs past its +timeout+.
|
|
26
|
+
class Timeout < Exception
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# A Robin::Job::ClassMethods#robin_retry_in block: seconds, :discard,
|
|
30
|
+
# :kill, or nil for the default backoff.
|
|
31
|
+
type retry_in = ^(Integer count, Exception error, job_hash payload) -> (seconds | :discard | :kill | nil)
|
|
32
|
+
|
|
33
|
+
# A Robin::Job::ClassMethods#robin_retries_exhausted block.
|
|
34
|
+
type retries_exhausted = ^(job_hash payload, Exception error) -> void
|
|
35
|
+
|
|
36
|
+
interface _InclusionCheck
|
|
37
|
+
def call: (Module base) -> void
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
self.@inclusion_checks: Array[_InclusionCheck]
|
|
41
|
+
|
|
42
|
+
# Callables run with each class about to include Robin::Job; one raises
|
|
43
|
+
# to reject the class.
|
|
44
|
+
def self.inclusion_checks: () -> Array[_InclusionCheck]
|
|
45
|
+
|
|
46
|
+
def self.append_features: (Module base) -> void
|
|
47
|
+
|
|
48
|
+
# Option hashes use string keys, and symbol values become strings.
|
|
49
|
+
def self.stringify: (Hash[Symbol | String, untyped] hash) -> Hash[String, untyped]
|
|
50
|
+
|
|
51
|
+
@interrupt_requested: bool?
|
|
52
|
+
|
|
53
|
+
# The job's id (set before #perform runs).
|
|
54
|
+
attr_accessor jid: String?
|
|
55
|
+
|
|
56
|
+
# The job as it was fetched (set before #perform runs).
|
|
57
|
+
attr_accessor payload: job_hash?
|
|
58
|
+
|
|
59
|
+
attr_accessor interrupt_requested: bool?
|
|
60
|
+
|
|
61
|
+
def logger: () -> Logger
|
|
62
|
+
|
|
63
|
+
def queue: () -> String?
|
|
64
|
+
|
|
65
|
+
def batch: () -> Batch?
|
|
66
|
+
|
|
67
|
+
def bid: () -> String?
|
|
68
|
+
|
|
69
|
+
# How many times this job has failed before.
|
|
70
|
+
def failures: () -> Integer
|
|
71
|
+
|
|
72
|
+
# The capsule running the job, or nil outside a server's workers.
|
|
73
|
+
def capsule: () -> Capsule?
|
|
74
|
+
|
|
75
|
+
# True once someone cancelled the job (or its batch).
|
|
76
|
+
def cancelled?: () -> bool
|
|
77
|
+
|
|
78
|
+
# Reports progress (0-100, or nil) with an optional message.
|
|
79
|
+
def progress!: (Integer | Float | nil percent, ?String? message, **untyped meta) -> void
|
|
80
|
+
|
|
81
|
+
# True while the server is shutting down.
|
|
82
|
+
def interrupted?: () -> bool
|
|
83
|
+
|
|
84
|
+
# What ClassMethods needs of the class it extends.
|
|
85
|
+
interface _JobClass
|
|
86
|
+
def name: () -> String?
|
|
87
|
+
|
|
88
|
+
# A job class's superclass may be a job class too.
|
|
89
|
+
def superclass: () -> untyped
|
|
90
|
+
|
|
91
|
+
def new: () -> untyped
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Class methods a job class gets (MyJob.perform_async...).
|
|
95
|
+
module ClassMethods : _JobClass
|
|
96
|
+
RESERVED: Array[String]
|
|
97
|
+
|
|
98
|
+
@robin_options: job_hash?
|
|
99
|
+
@robin_retry_in: retry_in?
|
|
100
|
+
@robin_retries_exhausted: retries_exhausted?
|
|
101
|
+
|
|
102
|
+
# Sets (with +opts+) and returns the class's options, string keys.
|
|
103
|
+
def robin_options: (?Hash[Symbol | String, untyped]? opts) -> job_hash
|
|
104
|
+
|
|
105
|
+
# Custom retry delay: the block receives (count, exception, payload)
|
|
106
|
+
# and returns seconds, :discard, :kill, or nil for the default.
|
|
107
|
+
def robin_retry_in: () { (Integer count, Exception error, job_hash payload) -> (seconds | :discard | :kill | nil) } -> retry_in
|
|
108
|
+
|
|
109
|
+
def robin_retry_in_block: () -> retry_in?
|
|
110
|
+
|
|
111
|
+
# Called when a job runs out of retries, with (payload, exception).
|
|
112
|
+
def robin_retries_exhausted: () { (job_hash payload, Exception error) -> void } -> retries_exhausted
|
|
113
|
+
|
|
114
|
+
def robin_retries_exhausted_block: () -> retries_exhausted?
|
|
115
|
+
|
|
116
|
+
# Per-push overrides: queue:, wait:, wait_until:, sync:, pool: and
|
|
117
|
+
# any robin_options.
|
|
118
|
+
def set: (Hash[Symbol | String, untyped] options) -> Setter
|
|
119
|
+
|
|
120
|
+
# Pushes the job; returns its jid, or nil if middleware stopped it.
|
|
121
|
+
def perform_async: (*untyped args) -> String?
|
|
122
|
+
|
|
123
|
+
def perform_in: (time interval, *untyped args) -> String?
|
|
124
|
+
|
|
125
|
+
def perform_at: (time time, *untyped args) -> String?
|
|
126
|
+
|
|
127
|
+
# Pushes a job per argument list; returns their jids.
|
|
128
|
+
def perform_bulk: (Array[Array[untyped]] args_list, ?batch_size: Integer, ?spread_interval: seconds?) -> Array[String?]
|
|
129
|
+
|
|
130
|
+
# Runs the job now, in this thread; returns what #perform returns.
|
|
131
|
+
def perform_inline: (*untyped args) -> untyped
|
|
132
|
+
|
|
133
|
+
def client_push: (job_hash item) -> String?
|
|
134
|
+
|
|
135
|
+
def robin_config: () -> Config
|
|
136
|
+
|
|
137
|
+
private
|
|
138
|
+
|
|
139
|
+
def validate_options!: (job_hash opts) -> void
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
DEFAULT_OPTIONS: job_hash
|
|
143
|
+
|
|
144
|
+
BULK_BATCH_SIZE: Integer
|
|
145
|
+
|
|
146
|
+
# Holds per-push overrides: MyJob.set(queue: "critical").perform_async
|
|
147
|
+
class Setter
|
|
148
|
+
@klass: ClassMethods
|
|
149
|
+
@options: job_hash
|
|
150
|
+
@pool: Redis::Pool?
|
|
151
|
+
|
|
152
|
+
def initialize: (ClassMethods klass, Hash[Symbol | String, untyped] options) -> void
|
|
153
|
+
|
|
154
|
+
def set: (Hash[Symbol | String, untyped] options) -> self
|
|
155
|
+
|
|
156
|
+
def perform_async: (*untyped args) -> untyped
|
|
157
|
+
|
|
158
|
+
def perform_in: (time interval, *untyped args) -> String?
|
|
159
|
+
|
|
160
|
+
def perform_at: (time interval, *untyped args) -> String?
|
|
161
|
+
|
|
162
|
+
def perform_bulk: (Array[Array[untyped]] args_list, ?batch_size: Integer, ?spread_interval: seconds?) -> Array[String?]
|
|
163
|
+
|
|
164
|
+
def perform_inline: (*untyped args) -> untyped
|
|
165
|
+
|
|
166
|
+
private
|
|
167
|
+
|
|
168
|
+
def via_pool: [T] () { () -> T } -> T
|
|
169
|
+
|
|
170
|
+
def item: (Array[untyped]? args) -> job_hash
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Pausing one job class across every queue (see Robin::JobClass).
|
|
3
|
+
class JobClass
|
|
4
|
+
type listing = {"name" => String, "parked" => Integer}
|
|
5
|
+
|
|
6
|
+
# Parks a fetched job if its class is still paused.
|
|
7
|
+
PARK: Redis::Script
|
|
8
|
+
|
|
9
|
+
# Moves parked jobs back onto the front of their queues.
|
|
10
|
+
UNPARK: Redis::Script
|
|
11
|
+
|
|
12
|
+
# The paused classes and their parked counts.
|
|
13
|
+
LIST: Redis::Script
|
|
14
|
+
|
|
15
|
+
# Each config's local copy of the paused classes, by its object_id.
|
|
16
|
+
self.@cache: Hash[Integer, Set[String]]
|
|
17
|
+
|
|
18
|
+
# The paused classes' names, sorted.
|
|
19
|
+
def self.paused: (?config: Config) -> Array[String]
|
|
20
|
+
|
|
21
|
+
# Each paused class: its name and how many of its jobs are parked.
|
|
22
|
+
def self.list: (?config: Config) -> Array[listing]
|
|
23
|
+
|
|
24
|
+
# The command that reads the paused classes.
|
|
25
|
+
def self.listing: (Config config) -> Redis::command
|
|
26
|
+
|
|
27
|
+
# Replaces the local copy of the paused classes.
|
|
28
|
+
def self.store: (Config config, Array[String] names) -> Set[String]
|
|
29
|
+
|
|
30
|
+
# The job's class (or the one it wraps) this process last saw paused.
|
|
31
|
+
def self.paused_class: (Config config, Hash[String, untyped] payload) -> String?
|
|
32
|
+
|
|
33
|
+
# Parks a fetched job of a paused class; whether it did.
|
|
34
|
+
def self.park: (Config config, Fetcher::UnitOfWork unit, String name) -> bool
|
|
35
|
+
|
|
36
|
+
# Notes a pause or unpause made by this process in its local copy.
|
|
37
|
+
def self.noted: (Config config, String name, bool paused) -> Set[String]?
|
|
38
|
+
|
|
39
|
+
attr_reader name: String
|
|
40
|
+
|
|
41
|
+
@config: Config
|
|
42
|
+
|
|
43
|
+
def initialize: (Module | String | Symbol klass, ?config: Config) -> void
|
|
44
|
+
|
|
45
|
+
def paused?: () -> bool
|
|
46
|
+
|
|
47
|
+
# How many of its jobs are parked.
|
|
48
|
+
def parked: () -> Integer
|
|
49
|
+
|
|
50
|
+
def pause!: () -> true
|
|
51
|
+
|
|
52
|
+
# Unpauses the class and puts its parked jobs back; how many.
|
|
53
|
+
def unpause!: (?chunk: Integer) -> Integer
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Per-thread logging context (class, jid, bid...), included in every log
|
|
3
|
+
# line written while a job runs.
|
|
4
|
+
module Context
|
|
5
|
+
KEY: Symbol
|
|
6
|
+
|
|
7
|
+
# This thread's context.
|
|
8
|
+
def self.current: () -> Hash[Symbol, untyped]
|
|
9
|
+
|
|
10
|
+
# Adds +hash+ to the context for the block.
|
|
11
|
+
def self.with: [T] (Hash[Symbol, untyped] hash) { () -> T } -> T
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Logs "start", then "done" or "fail", around each job.
|
|
15
|
+
class JobLogger
|
|
16
|
+
# Text log lines, with the context as key=value pairs.
|
|
17
|
+
class Formatter < ::Logger::Formatter
|
|
18
|
+
def call: (String severity, Time time, String? progname, untyped message) -> String
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# One-line JSON per log entry.
|
|
22
|
+
class JSONFormatter < ::Logger::Formatter
|
|
23
|
+
def call: (String severity, Time time, String? progname, untyped message) -> String
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
def clean: (untyped value) -> untyped
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The logging context of a job: its class, jid, and bid and tags when
|
|
31
|
+
# it has them.
|
|
32
|
+
def self.context_for: (job_hash payload) -> Hash[Symbol, untyped]
|
|
33
|
+
|
|
34
|
+
# This thread's id in log lines.
|
|
35
|
+
def self.tid: () -> String
|
|
36
|
+
|
|
37
|
+
@config: Config
|
|
38
|
+
|
|
39
|
+
def initialize: (Config config) -> void
|
|
40
|
+
|
|
41
|
+
LEVELS: Array[String]
|
|
42
|
+
|
|
43
|
+
ENABLED: Hash[String, Symbol]
|
|
44
|
+
|
|
45
|
+
# Runs the job (the block), logging around it; returns the block's value.
|
|
46
|
+
def call: [T] (job_hash payload) { () -> T } -> T
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def level_for: (job_hash payload) -> String?
|
|
51
|
+
|
|
52
|
+
def log: (String level, String message) -> void
|
|
53
|
+
|
|
54
|
+
def elapsed: (Float start) -> String
|
|
55
|
+
end
|
|
56
|
+
end
|