robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Per-job-class execution metrics.
|
|
3
|
+
module Metrics
|
|
4
|
+
# One class's counts since the last drain: successes, failures, total
|
|
5
|
+
# milliseconds of the successful runs, and runs per histogram bucket.
|
|
6
|
+
type counts = {s: Integer, f: Integer, ms: Float, hist: Hash[Integer, Integer]}
|
|
7
|
+
|
|
8
|
+
# Each job class's counts, by its (display) name.
|
|
9
|
+
type jobs = Hash[String, counts]
|
|
10
|
+
|
|
11
|
+
# What Collector#drain returns: per-class counts, processed, failed.
|
|
12
|
+
type drained = [jobs, Integer, Integer]
|
|
13
|
+
|
|
14
|
+
# A queue's size and latency (seconds), as sampled.
|
|
15
|
+
type queue_sample = {size: Integer, latency: Float}
|
|
16
|
+
|
|
17
|
+
# Upper bounds (ms) of the histogram buckets; the last is open-ended.
|
|
18
|
+
BUCKETS: Array[Integer | Float]
|
|
19
|
+
|
|
20
|
+
MINUTE_TTL: Integer
|
|
21
|
+
|
|
22
|
+
DAILY_TOTALS_TTL: Integer
|
|
23
|
+
|
|
24
|
+
# The histogram bucket of a run of +ms+ milliseconds.
|
|
25
|
+
def self.bucket_for: (Integer | Float ms) -> Integer
|
|
26
|
+
|
|
27
|
+
def self.minute_stamp: (Time time) -> String
|
|
28
|
+
|
|
29
|
+
def self.hour_stamp: (Time time) -> String
|
|
30
|
+
|
|
31
|
+
# Records queue sizes and latencies (leader only, once a minute).
|
|
32
|
+
def self.sample_queues: (Config config, Redis::Connection conn, ?Time time) -> void
|
|
33
|
+
|
|
34
|
+
# Records counts per thread, in stripes, and drains them together.
|
|
35
|
+
class Collector
|
|
36
|
+
# One thread's counts.
|
|
37
|
+
class Stripe
|
|
38
|
+
@mutex: Thread::Mutex
|
|
39
|
+
@jobs: jobs
|
|
40
|
+
@processed: Integer
|
|
41
|
+
@failed: Integer
|
|
42
|
+
|
|
43
|
+
attr_reader thread: Thread
|
|
44
|
+
|
|
45
|
+
def initialize: (Thread thread) -> void
|
|
46
|
+
|
|
47
|
+
def record: (String klass, Integer | Float ms, bool ok) -> void
|
|
48
|
+
|
|
49
|
+
# Adds counts drained earlier.
|
|
50
|
+
def add: (drained drained) -> void
|
|
51
|
+
|
|
52
|
+
def drain: () -> drained
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def reset: () -> void
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def self.entry: (jobs jobs, String klass) -> counts
|
|
60
|
+
|
|
61
|
+
# Adds +from+'s per-class counts to +into+'s.
|
|
62
|
+
def self.merge: (jobs into, jobs from) -> void
|
|
63
|
+
|
|
64
|
+
@mutex: Thread::Mutex
|
|
65
|
+
@stripes: Array[Stripe]
|
|
66
|
+
@key: Symbol
|
|
67
|
+
|
|
68
|
+
def initialize: () -> void
|
|
69
|
+
|
|
70
|
+
def record: (String klass, Integer | Float ms, bool ok) -> void
|
|
71
|
+
|
|
72
|
+
# [per-class counts, processed, failed] since the last drain.
|
|
73
|
+
def drain: () -> drained
|
|
74
|
+
|
|
75
|
+
# Puts drained data back (used when a flush fails).
|
|
76
|
+
def restore: (drained drained) -> void
|
|
77
|
+
|
|
78
|
+
# Writes what was recorded since the last flush in one pipeline.
|
|
79
|
+
def flush: (Config config, Redis::Connection conn, ?Time time) -> void
|
|
80
|
+
|
|
81
|
+
# Queues the commands writing what was recorded since the last flush
|
|
82
|
+
# on +pipe+; returns what they write.
|
|
83
|
+
def queue: (Config config, Redis::Pipeline pipe, ?Time time) -> drained
|
|
84
|
+
|
|
85
|
+
# Reports queued metrics to StatsD once they're written.
|
|
86
|
+
def flushed: (Config config, drained drained) -> void
|
|
87
|
+
|
|
88
|
+
private
|
|
89
|
+
|
|
90
|
+
def stripe: () -> Stripe
|
|
91
|
+
|
|
92
|
+
def write: (Config config, Redis::Pipeline p, Time time, drained drained) -> void
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Query helpers for the web UI and API. Their results are JSON for the
|
|
96
|
+
# dashboard.
|
|
97
|
+
class Query
|
|
98
|
+
GRANULARITIES: Hash[Symbol, [Integer, String, Symbol]]
|
|
99
|
+
|
|
100
|
+
@config: Config
|
|
101
|
+
|
|
102
|
+
def initialize: (?Config config) -> void
|
|
103
|
+
|
|
104
|
+
# {"classes" => per-class totals, "series" => per-minute points}.
|
|
105
|
+
def recent: (?minutes: Integer, ?now: Time) -> Hash[String, untyped]
|
|
106
|
+
|
|
107
|
+
# The same, hourly.
|
|
108
|
+
def history: (?hours: Integer, ?now: Time) -> Hash[String, untyped]
|
|
109
|
+
|
|
110
|
+
# Queue size/latency maxima per minute or hour: {"t" => stamp,
|
|
111
|
+
# "queues" => {queue => {"size" => n, "latency" => n}}} each.
|
|
112
|
+
def queue_history: (?granularity: Symbol, ?points: Integer, ?now: Time) -> Array[Hash[String, untyped]]
|
|
113
|
+
|
|
114
|
+
# Latency histogram for one class: {"le" => bound or nil, "count" => n} each.
|
|
115
|
+
def histogram: (String klass, ?minutes: Integer, ?now: Time) -> Array[Hash[String, Integer | Float | nil]]
|
|
116
|
+
|
|
117
|
+
private
|
|
118
|
+
|
|
119
|
+
def summarize: (Array[String] stamps, Array[Redis::reply] rows) -> Hash[String, untyped]
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Middleware wraps job pushes (client side) and job execution (server
|
|
3
|
+
# side), Rack style. A middleware is a class; the chain builds it (with
|
|
4
|
+
# the arguments given to Chain#add) and calls it:
|
|
5
|
+
#
|
|
6
|
+
# Client: call(job_class, payload, queue, redis) { ... } (see _Client)
|
|
7
|
+
# Server: call(job, payload, queue) { ... } (see _Server)
|
|
8
|
+
module Middleware
|
|
9
|
+
# A client middleware: yields to push the job (return what the block
|
|
10
|
+
# returns), or returns nil instead to stop the push. +job_class+ is the
|
|
11
|
+
# job's class name and +redis+ the pool the job goes to.
|
|
12
|
+
interface _Client
|
|
13
|
+
def call: (String job_class, job_hash payload, String queue, Redis::Pool redis) { () -> job_hash? } -> job_hash?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# A server middleware: yields to run the job. +job+ is the job's
|
|
17
|
+
# instance (nil for a job whose class isn't a Robin::Job).
|
|
18
|
+
interface _Server
|
|
19
|
+
def call: (untyped job, job_hash payload, String queue) { () -> untyped } -> untyped
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Any middleware: what the chain calls, with the arguments it was invoked
|
|
23
|
+
# with.
|
|
24
|
+
interface _Middleware
|
|
25
|
+
def call: (*untyped args) { () -> untyped } -> untyped
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Gives a middleware the Robin config (set as it's built).
|
|
29
|
+
module ConfigAware
|
|
30
|
+
attr_accessor config: Config
|
|
31
|
+
|
|
32
|
+
def redis: [T] () { (Redis::Connection conn) -> T } -> T
|
|
33
|
+
|
|
34
|
+
def logger: () -> Logger
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# One instance serves every job (on every thread).
|
|
38
|
+
module Stateless
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
class Entry
|
|
42
|
+
# A middleware class may define +applies?+, so it's duck typed here.
|
|
43
|
+
@klass: untyped
|
|
44
|
+
@config: Config
|
|
45
|
+
@args: Array[untyped]
|
|
46
|
+
@stateless: bool
|
|
47
|
+
@filtered: bool
|
|
48
|
+
@instance: _Middleware?
|
|
49
|
+
|
|
50
|
+
# The middleware's class. One that defines +applies?+ (a class method
|
|
51
|
+
# taking +call+'s arguments) is skipped for jobs it returns false for.
|
|
52
|
+
def klass: () -> Class
|
|
53
|
+
|
|
54
|
+
def initialize: (Config config, Class klass, Array[untyped] args) -> void
|
|
55
|
+
|
|
56
|
+
def applies?: (Array[untyped] args) -> bool
|
|
57
|
+
|
|
58
|
+
# The instance to call.
|
|
59
|
+
def build: () -> _Middleware
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def instantiate: () -> _Middleware
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
class Chain
|
|
67
|
+
include Enumerable[Entry]
|
|
68
|
+
|
|
69
|
+
@config: Config
|
|
70
|
+
|
|
71
|
+
def initialize: (Config config) -> void
|
|
72
|
+
|
|
73
|
+
# Frozen: change the chain with its methods.
|
|
74
|
+
attr_reader entries: Array[Entry]
|
|
75
|
+
|
|
76
|
+
def each: () { (Entry entry) -> void } -> Array[Entry]
|
|
77
|
+
|
|
78
|
+
def add: (Class klass, *untyped args) -> self
|
|
79
|
+
|
|
80
|
+
def prepend: (Class klass, *untyped args) -> self
|
|
81
|
+
|
|
82
|
+
def insert_before: (Class target, Class klass, *untyped args) -> self
|
|
83
|
+
|
|
84
|
+
def insert_after: (Class target, Class klass, *untyped args) -> self
|
|
85
|
+
|
|
86
|
+
def remove: (Class klass) -> self
|
|
87
|
+
|
|
88
|
+
def exists?: (Class klass) -> bool
|
|
89
|
+
|
|
90
|
+
def empty?: () -> bool
|
|
91
|
+
|
|
92
|
+
def clear: () -> self
|
|
93
|
+
|
|
94
|
+
# Invokes the chain around the block, outermost middleware first;
|
|
95
|
+
# returns the block's value, or what a middleware that halted the
|
|
96
|
+
# chain returned.
|
|
97
|
+
def invoke: (*untyped args) { () -> untyped } -> untyped
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def call_from: (Array[Entry] entries, Integer index, Array[untyped] args, ^() -> untyped block) -> untyped
|
|
102
|
+
|
|
103
|
+
def without: (Class klass) -> Array[Entry]
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# OpenTelemetry tracing for pushing and running jobs (require
|
|
3
|
+
# "robin/opentelemetry").
|
|
4
|
+
module OpenTelemetry
|
|
5
|
+
# What Robin uses of an OpenTelemetry tracer provider and its tracers
|
|
6
|
+
# (Robin doesn't depend on OpenTelemetry, so its signatures can't name
|
|
7
|
+
# them).
|
|
8
|
+
interface _TracerProvider
|
|
9
|
+
def tracer: (?String? name, ?String? version) -> _Tracer
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
interface _Tracer
|
|
13
|
+
def in_span: [T] (String name, ?attributes: Hash[String, untyped]?, ?links: Array[untyped]?, ?kind: Symbol?) { () -> T } -> T
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Adds the tracing middleware to +config+; +parent+ is :child (a run's
|
|
17
|
+
# span is a child of its push's) or :link (linked to it).
|
|
18
|
+
def self.install: (Config config, ?tracer_provider: _TracerProvider, ?parent: :child | :link) -> Config
|
|
19
|
+
|
|
20
|
+
# A span's attributes for a job.
|
|
21
|
+
def self.attributes: (job_hash payload, String queue, String operation) -> Hash[String, untyped]
|
|
22
|
+
|
|
23
|
+
# Opens a "publish <queue>" span around each push.
|
|
24
|
+
class ClientMiddleware
|
|
25
|
+
include Middleware::Stateless
|
|
26
|
+
|
|
27
|
+
@tracer: _Tracer
|
|
28
|
+
|
|
29
|
+
def initialize: (_Tracer tracer) -> void
|
|
30
|
+
|
|
31
|
+
def call: (String _job_class, job_hash payload, String queue, Redis::Pool _redis) { () -> job_hash? } -> job_hash?
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Opens a "process <queue>" span around each run.
|
|
35
|
+
class ServerMiddleware
|
|
36
|
+
include Middleware::Stateless
|
|
37
|
+
|
|
38
|
+
@tracer: _Tracer
|
|
39
|
+
@parent: :child | :link
|
|
40
|
+
|
|
41
|
+
def initialize: (_Tracer tracer, :child | :link parent) -> void
|
|
42
|
+
|
|
43
|
+
def call: (untyped _job, job_hash payload, String queue) { () -> untyped } -> untyped
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
# Runs the block in an empty OpenTelemetry context (yielded).
|
|
48
|
+
def detached: [T] () { (untyped context) -> T } -> T
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Periodic (cron) jobs: registered in config.periodic blocks, enqueued by
|
|
3
|
+
# the cluster leader.
|
|
4
|
+
module Periodic
|
|
5
|
+
# A registered periodic job, as stored: "cron", "class", "queue",
|
|
6
|
+
# "args", "tz", "description" and "options" (its job options).
|
|
7
|
+
type entry = Hash[String, untyped]
|
|
8
|
+
|
|
9
|
+
# A periodic job as Periodic.list shows it (JSON-ready).
|
|
10
|
+
type listing = Hash[String, untyped]
|
|
11
|
+
|
|
12
|
+
# A tick to claim or that was claimed: [entry id, due time (epoch ms)].
|
|
13
|
+
type tick = [String, Integer]
|
|
14
|
+
|
|
15
|
+
HISTORY: Integer
|
|
16
|
+
MAX_SLEEP: Integer
|
|
17
|
+
CLAIM_TTL: Integer
|
|
18
|
+
CLAIM: Redis::Script
|
|
19
|
+
|
|
20
|
+
# Raised by Periodic.verify! with every problem it found.
|
|
21
|
+
class Invalid < Robin::Error
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Collects registrations from config.periodic blocks.
|
|
25
|
+
class Registry
|
|
26
|
+
@problems: Array[String]?
|
|
27
|
+
|
|
28
|
+
# The registered entries, by id.
|
|
29
|
+
attr_reader entries: Hash[String, entry]
|
|
30
|
+
|
|
31
|
+
def initialize: (?Array[String]? problems) -> void
|
|
32
|
+
|
|
33
|
+
# Registers a periodic job; returns its id (nil when a bad one was
|
|
34
|
+
# noted as a problem instead). Other keywords are job options.
|
|
35
|
+
def register: (String cron, job_class | Symbol klass, ?queue: String | Symbol | nil, ?args: Array[untyped], ?tz: String?, ?description: String?, **untyped options) -> String?
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Checks periodic registrations (the block, or every block of +config+);
|
|
39
|
+
# raises Invalid listing every problem, else returns the entries.
|
|
40
|
+
def self.verify!: (?Config config) ?{ (Registry cron) -> void } -> Hash[String, entry]
|
|
41
|
+
|
|
42
|
+
# Starts the leader's periodic thread (Launcher#run calls it).
|
|
43
|
+
def self.attach: (Launcher launcher) -> Thread?
|
|
44
|
+
|
|
45
|
+
# Every periodic job; with +history+, its options and latest runs too.
|
|
46
|
+
def self.list: (?config: Config, ?history: Integer?) -> Array[listing]
|
|
47
|
+
|
|
48
|
+
# A periodic job's latest runs ({"at", "jid", "manual"}), newest first.
|
|
49
|
+
def self.history: (String id, ?config: Config, ?limit: Integer) -> Array[Hash[String, untyped]]
|
|
50
|
+
|
|
51
|
+
# Enqueues the job right away; returns its jid.
|
|
52
|
+
def self.enqueue_now: (String id, ?config: Config) -> String?
|
|
53
|
+
|
|
54
|
+
def self.pause!: (String id, ?config: Config) -> true
|
|
55
|
+
|
|
56
|
+
def self.unpause!: (String id, ?config: Config) -> true
|
|
57
|
+
|
|
58
|
+
# Enqueues the entries whose latest tick hasn't been claimed yet;
|
|
59
|
+
# returns their jids.
|
|
60
|
+
def self.tick: (Config config, Hash[String, entry] entries, Hash[String, Cron] crons, ?Time now, ?Hash[String, Integer] claimed) -> Array[String?]
|
|
61
|
+
|
|
62
|
+
# Writes the entries to Redis as the periodic jobs, arming new ones.
|
|
63
|
+
def self.write_registry: (Config config, Hash[String, entry] entries) -> void
|
|
64
|
+
|
|
65
|
+
private def self.claim: (Config config, Array[tick] dues, Hash[String, Cron] crons, Time now, Hash[String, Integer] claimed) -> Array[tick]
|
|
66
|
+
|
|
67
|
+
private def self.run_loop: (Launcher launcher, Hash[String, entry] entries, Hash[String, Cron] crons) -> void
|
|
68
|
+
|
|
69
|
+
private def self.seconds_until_next: (Hash[String, Cron] crons, Time now) -> (Float | Integer)
|
|
70
|
+
|
|
71
|
+
private def self.entry_problems: (entry entry) -> Array[String]
|
|
72
|
+
|
|
73
|
+
private def self.job_class: (String name) -> untyped
|
|
74
|
+
|
|
75
|
+
private def self.push_job: (Config config, String id, entry entry) -> String?
|
|
76
|
+
|
|
77
|
+
private def self.record: (Config config, Array[[String, Integer, String?]] pushed, ?manual: bool) -> void
|
|
78
|
+
|
|
79
|
+
private def self.parse: (entry entry) -> Cron?
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# A worker thread: fetch a job, run it through middleware, record the
|
|
3
|
+
# outcome, repeat.
|
|
4
|
+
class Processor
|
|
5
|
+
# What a worker is running: [queue, raw payload, started at (epoch ms),
|
|
6
|
+
# in-flight key].
|
|
7
|
+
type work = [String, String, Integer, String]
|
|
8
|
+
|
|
9
|
+
MIN_BACKOFF: Float
|
|
10
|
+
|
|
11
|
+
MAX_BACKOFF: Integer
|
|
12
|
+
|
|
13
|
+
@manager: Manager
|
|
14
|
+
@config: Config
|
|
15
|
+
@fetcher: Fetcher
|
|
16
|
+
@slot: Integer
|
|
17
|
+
@retrier: Retrier
|
|
18
|
+
@job_logger: JobLogger
|
|
19
|
+
@wrapper: job_wrapper?
|
|
20
|
+
@failures: Integer
|
|
21
|
+
@backoff: seconds
|
|
22
|
+
@retired: bool
|
|
23
|
+
@done: bool?
|
|
24
|
+
@outage_started: Float?
|
|
25
|
+
# nil outside #execute
|
|
26
|
+
@batch_failure: Batch::HeldFailure?
|
|
27
|
+
|
|
28
|
+
attr_reader capsule: Capsule
|
|
29
|
+
|
|
30
|
+
# The worker thread (nil until #start).
|
|
31
|
+
attr_reader thread: Thread?
|
|
32
|
+
|
|
33
|
+
# The job it's running, if any.
|
|
34
|
+
attr_reader job: untyped
|
|
35
|
+
|
|
36
|
+
# The in-flight key of the last finished job, until its acknowledgement
|
|
37
|
+
# goes out with the next fetch.
|
|
38
|
+
attr_reader unacked: String?
|
|
39
|
+
|
|
40
|
+
# The running job, or nil; kept up to date by the launcher.
|
|
41
|
+
attr_accessor work: work?
|
|
42
|
+
|
|
43
|
+
# When (monotonic seconds) it last finished a job, or nil.
|
|
44
|
+
attr_accessor finished_at: Float?
|
|
45
|
+
|
|
46
|
+
# How many jobs it has finished.
|
|
47
|
+
attr_accessor finished: Integer
|
|
48
|
+
|
|
49
|
+
def initialize: (Manager manager, Capsule capsule, Fetcher fetcher, Integer slot) -> void
|
|
50
|
+
|
|
51
|
+
# Starts the worker thread, after +delay+ seconds.
|
|
52
|
+
def start: (seconds delay) -> self
|
|
53
|
+
|
|
54
|
+
# Stop after the current job.
|
|
55
|
+
def terminate: () -> void
|
|
56
|
+
|
|
57
|
+
# Stop after the current job, without interrupting it.
|
|
58
|
+
def retire: () -> void
|
|
59
|
+
|
|
60
|
+
def retired?: () -> bool
|
|
61
|
+
|
|
62
|
+
def alive?: () -> bool?
|
|
63
|
+
|
|
64
|
+
# Hard stop: interrupt the running job.
|
|
65
|
+
def kill: () -> void
|
|
66
|
+
|
|
67
|
+
# Fetches the next job and runs it, on the calling thread; whether
|
|
68
|
+
# there was one.
|
|
69
|
+
def work_once: () -> bool
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def run: (seconds delay) -> void
|
|
74
|
+
|
|
75
|
+
def fetch: () -> Fetcher::UnitOfWork?
|
|
76
|
+
|
|
77
|
+
def process: (Fetcher::UnitOfWork unit) -> void
|
|
78
|
+
|
|
79
|
+
def complete: (Fetcher::UnitOfWork unit, job_hash payload) -> void
|
|
80
|
+
|
|
81
|
+
def acknowledge_unacked: () -> void
|
|
82
|
+
|
|
83
|
+
def release_fetched: () -> void
|
|
84
|
+
|
|
85
|
+
def execute: (Class job_class, job_hash payload, Fetcher::UnitOfWork unit) -> void
|
|
86
|
+
|
|
87
|
+
def wrapped: (job_hash payload, ^() -> void run_job) -> void
|
|
88
|
+
|
|
89
|
+
# Does #process hand +exception+ to the retrier?
|
|
90
|
+
def retried?: (Exception exception) -> bool
|
|
91
|
+
|
|
92
|
+
def record_batch_failure: () -> void
|
|
93
|
+
|
|
94
|
+
def take_batch_failure: () -> Batch::HeldFailure?
|
|
95
|
+
|
|
96
|
+
# Runs the job's #perform: what it returns.
|
|
97
|
+
def perform: (untyped job, job_hash payload) -> untyped
|
|
98
|
+
|
|
99
|
+
def interruptible: [T] () { () -> T } -> T
|
|
100
|
+
|
|
101
|
+
def handle_failure: (Fetcher::UnitOfWork unit, job_hash payload, Class? job_class, Exception exception) -> void
|
|
102
|
+
|
|
103
|
+
def constantize: (String name) -> untyped
|
|
104
|
+
|
|
105
|
+
def monotonic: () -> Float
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Profiles single job runs with Vernier (see the "profile" job option).
|
|
3
|
+
module Profiler
|
|
4
|
+
TTL: Integer
|
|
5
|
+
|
|
6
|
+
KEEP: Integer
|
|
7
|
+
|
|
8
|
+
MAX_BYTES: Integer
|
|
9
|
+
|
|
10
|
+
LOCK: Thread::Mutex
|
|
11
|
+
|
|
12
|
+
# nil until the first check.
|
|
13
|
+
self.@available: bool
|
|
14
|
+
|
|
15
|
+
# Loads Vernier once, warning when it can't.
|
|
16
|
+
def self.available?: (Config config) -> bool
|
|
17
|
+
|
|
18
|
+
# Runs the block under the profiler and stores the result. Returns the
|
|
19
|
+
# block's value.
|
|
20
|
+
def self.profile: [T] (Config config, job_hash payload) { () -> T } -> T
|
|
21
|
+
|
|
22
|
+
# A running collector, or nil if one can't start.
|
|
23
|
+
# What Robin uses of a Vernier::Collector (Robin doesn't depend on
|
|
24
|
+
# Vernier, so its signatures can't name it).
|
|
25
|
+
interface _Collector
|
|
26
|
+
def stop: () -> _Profile
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
interface _Profile
|
|
30
|
+
# The profile in the Firefox Profiler format (JSON).
|
|
31
|
+
def to_gecko: () -> String
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.start: (Config config) -> _Collector?
|
|
35
|
+
|
|
36
|
+
def self.record: (Config config, job_hash payload, _Collector collector, Integer started, bool ok) -> void
|
|
37
|
+
|
|
38
|
+
# Stored profiles, newest first (without their data).
|
|
39
|
+
def self.list: (?config: Config) -> Array[Hash[String, untyped]]
|
|
40
|
+
|
|
41
|
+
# The profile JSON for +jid+, or nil once it has expired.
|
|
42
|
+
def self.data: (String jid, ?config: Config) -> String?
|
|
43
|
+
|
|
44
|
+
# Deletes the profile of +jid+; true if there was one.
|
|
45
|
+
def self.delete: (String jid, ?config: Config) -> bool
|
|
46
|
+
|
|
47
|
+
# [raw, parsed] pairs, newest first.
|
|
48
|
+
private def self.entries: (Config config) -> Array[[String, Hash[String, untyped]]]
|
|
49
|
+
|
|
50
|
+
# Profiles jobs pushed with a "profile" option.
|
|
51
|
+
class ServerMiddleware
|
|
52
|
+
include Middleware::ConfigAware
|
|
53
|
+
include Middleware::Stateless
|
|
54
|
+
|
|
55
|
+
def self.applies?: (untyped _job, job_hash payload, String _queue) -> bool
|
|
56
|
+
|
|
57
|
+
def call: (untyped _job, job_hash payload, String _queue) { () -> untyped } -> untyped
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Live progress reporting for long jobs, shown on the web UI's Busy page.
|
|
3
|
+
module Progress
|
|
4
|
+
# What #report stores: "pct" (0-100, or nil when unknown), "msg", "at"
|
|
5
|
+
# (epoch ms) and, with any, "meta".
|
|
6
|
+
type entry = Hash[String, untyped]
|
|
7
|
+
|
|
8
|
+
TTL: Integer
|
|
9
|
+
|
|
10
|
+
def self.report: (String? jid, Integer | Float | nil percent, ?String? message, ?Hash[String | Symbol, untyped] meta, ?config: Config) -> entry
|
|
11
|
+
|
|
12
|
+
# What #report stores.
|
|
13
|
+
def self.entry: (Integer | Float | nil percent, ?String? message, ?Hash[String | Symbol, untyped] meta) -> entry
|
|
14
|
+
|
|
15
|
+
# The command #report sends.
|
|
16
|
+
def self.set_command: (Config config, String? jid, entry data) -> Redis::command
|
|
17
|
+
|
|
18
|
+
def self.fetch: (String jid, ?config: Config) -> entry?
|
|
19
|
+
|
|
20
|
+
def self.fetch_many: (Array[String] jids, ?config: Config) -> Hash[String, entry]
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
module Rails
|
|
3
|
+
# Carries ActiveSupport::CurrentAttributes (named by class) from where a
|
|
4
|
+
# job is pushed into the job.
|
|
5
|
+
module CurrentAttributes
|
|
6
|
+
# Class names: one, or several.
|
|
7
|
+
type names = String | Symbol | Array[String | Symbol]
|
|
8
|
+
|
|
9
|
+
# The job field holding the attributes of the class at +index+.
|
|
10
|
+
def self.key_at: (Integer index) -> String
|
|
11
|
+
|
|
12
|
+
# [field, class name] for each configured class.
|
|
13
|
+
def self.fields: (names names) -> Array[[String, String]]
|
|
14
|
+
|
|
15
|
+
# Client middleware: saves the attributes into jobs being pushed.
|
|
16
|
+
class Save
|
|
17
|
+
include Robin::Middleware::Stateless
|
|
18
|
+
|
|
19
|
+
@fields: Array[[String, String]]
|
|
20
|
+
|
|
21
|
+
def initialize: (names names) -> void
|
|
22
|
+
|
|
23
|
+
def call: (String _job_class, job_hash payload, String _queue, Redis::Pool _redis) { () -> job_hash? } -> job_hash?
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Server middleware: sets the saved attributes while a job runs.
|
|
27
|
+
class Load
|
|
28
|
+
include Robin::Middleware::Stateless
|
|
29
|
+
|
|
30
|
+
@fields: Array[[String, String]]
|
|
31
|
+
|
|
32
|
+
def initialize: (names names) -> void
|
|
33
|
+
|
|
34
|
+
def call: (untyped _job, job_hash payload, String _queue) { () -> untyped } -> untyped
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def set: (Array[[String, String]] fields, job_hash payload) { () -> untyped } -> untyped
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def self.install: (Config config, names names) -> void
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
module Rails
|
|
3
|
+
# A Robin error handler that reports to the Rails error reporter
|
|
4
|
+
# (Rails.error).
|
|
5
|
+
# (A Robin::_ErrorHandler.)
|
|
6
|
+
class ErrorReporter
|
|
7
|
+
# What it reports to: Rails.error (an ActiveSupport::ErrorReporter).
|
|
8
|
+
interface _Reporter
|
|
9
|
+
def report: (Exception error, handled: bool, source: String, context: Hash[Symbol, untyped]) -> void
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
JOB_FAILED: String
|
|
13
|
+
|
|
14
|
+
@reporter: _Reporter
|
|
15
|
+
|
|
16
|
+
def initialize: (_Reporter reporter) -> void
|
|
17
|
+
|
|
18
|
+
def call: (Exception exception, Hash[Symbol, untyped] context, Config _config) -> void
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
module Rails
|
|
3
|
+
# A Config#push_deferrer holding jobs pushed inside Active Record
|
|
4
|
+
# transactions until they all commit.
|
|
5
|
+
module TransactionalPush
|
|
6
|
+
# The fiber-local key of [the transaction, the deliveries waiting on it].
|
|
7
|
+
WAITING: Symbol
|
|
8
|
+
|
|
9
|
+
# Whether it took over delivery (false without Active Record 7.2+).
|
|
10
|
+
def self.call: (Array[job_hash] _payloads) { () -> void } -> bool
|
|
11
|
+
|
|
12
|
+
# The innermost open transaction on the primary database, if any.
|
|
13
|
+
private def self.current_transaction: () -> untyped
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
data/sig/robin/rails.rbs
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Rails integration for Robin: a Railtie, the ActiveJob adapter,
|
|
3
|
+
# generators, Rails.error reporting and transaction-aware pushes. (The
|
|
4
|
+
# signatures of the parts that subclass Rails' and Puma's classes aren't
|
|
5
|
+
# in the gem, so that checking against it doesn't need Rails' or Puma's.)
|
|
6
|
+
module Rails
|
|
7
|
+
end
|
|
8
|
+
end
|