robin 0.1.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +372 -0
- data/COPYING +674 -0
- data/LICENSE +165 -0
- data/README.md +1490 -0
- data/assets/favicon.svg +31 -0
- data/assets/robin-logo.svg +37 -0
- data/assets/robin-mono.svg +12 -0
- data/assets/robin-wordmark.svg +4 -0
- data/assets/robin.svg +34 -0
- data/exe/robin +5 -0
- data/ext/java/robin_resp/Blocking.java +27 -0
- data/ext/java/robin_resp/NativeIO.java +229 -0
- data/ext/java/robin_resp/NativeRESP.java +279 -0
- data/ext/java/robin_resp/NativeSlots.java +133 -0
- data/ext/java/robin_resp/RobinRespService.java +32 -0
- data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
- data/lib/generators/robin/install/install_generator.rb +39 -0
- data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
- data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
- data/lib/generators/robin/job/job_generator.rb +78 -0
- data/lib/generators/robin/job/templates/job.rb.tt +13 -0
- data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
- data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
- data/lib/puma/plugin/robin.rb +131 -0
- data/lib/robin/active_job.rb +144 -0
- data/lib/robin/api.rb +777 -0
- data/lib/robin/batch.rb +700 -0
- data/lib/robin/cancellation.rb +62 -0
- data/lib/robin/cli.rb +325 -0
- data/lib/robin/client.rb +372 -0
- data/lib/robin/config.rb +461 -0
- data/lib/robin/cron.rb +325 -0
- data/lib/robin/debounce.rb +108 -0
- data/lib/robin/deploy.rb +65 -0
- data/lib/robin/embedded.rb +53 -0
- data/lib/robin/encryption.rb +147 -0
- data/lib/robin/events.rb +60 -0
- data/lib/robin/expiry.rb +41 -0
- data/lib/robin/fetcher.rb +441 -0
- data/lib/robin/gvl.rb +62 -0
- data/lib/robin/health.rb +76 -0
- data/lib/robin/iterable_job.rb +294 -0
- data/lib/robin/job.rb +258 -0
- data/lib/robin/job_class.rb +157 -0
- data/lib/robin/job_logger.rb +121 -0
- data/lib/robin/launcher.rb +658 -0
- data/lib/robin/leader.rb +86 -0
- data/lib/robin/limiter.rb +639 -0
- data/lib/robin/manager.rb +125 -0
- data/lib/robin/metrics.rb +303 -0
- data/lib/robin/middleware.rb +165 -0
- data/lib/robin/opentelemetry.rb +88 -0
- data/lib/robin/periodic.rb +327 -0
- data/lib/robin/processor.rb +317 -0
- data/lib/robin/profiler.rb +140 -0
- data/lib/robin/progress.rb +46 -0
- data/lib/robin/rails/app_loader.rb +23 -0
- data/lib/robin/rails/current_attributes.rb +88 -0
- data/lib/robin/rails/error_reporter.rb +31 -0
- data/lib/robin/rails/notifications.rb +22 -0
- data/lib/robin/rails/railtie.rb +87 -0
- data/lib/robin/rails/transactional_push.rb +49 -0
- data/lib/robin/rails.rb +23 -0
- data/lib/robin/recovery.rb +135 -0
- data/lib/robin/redis/connection.rb +639 -0
- data/lib/robin/redis/pool.rb +190 -0
- data/lib/robin/redis/resp.rb +77 -0
- data/lib/robin/reliable_push.rb +153 -0
- data/lib/robin/retrier.rb +284 -0
- data/lib/robin/scheduler.rb +102 -0
- data/lib/robin/scripts.rb +331 -0
- data/lib/robin/statsd.rb +135 -0
- data/lib/robin/swarm.rb +319 -0
- data/lib/robin/systemd.rb +47 -0
- data/lib/robin/testing.rb +158 -0
- data/lib/robin/top.rb +178 -0
- data/lib/robin/unique.rb +198 -0
- data/lib/robin/version.rb +5 -0
- data/lib/robin/warmup.rb +94 -0
- data/lib/robin/web/api.rb +580 -0
- data/lib/robin/web/batch_status.rb +44 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
- data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
- data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
- data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
- data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
- data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
- data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
- data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
- data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
- data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
- data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
- data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
- data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
- data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
- data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
- data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
- data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
- data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
- data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
- data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
- data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
- data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
- data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
- data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
- data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
- data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
- data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
- data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
- data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
- data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
- data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
- data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
- data/lib/robin/web/dist/index.html +19 -0
- data/lib/robin/web/extension.rb +193 -0
- data/lib/robin/web/permissions.rb +40 -0
- data/lib/robin/web/prometheus.rb +79 -0
- data/lib/robin/web/router.rb +150 -0
- data/lib/robin/web/static.rb +110 -0
- data/lib/robin/web.rb +196 -0
- data/lib/robin.rb +261 -0
- data/lib/robin_resp/robin_resp.jar +0 -0
- data/sig/manifest.yaml +9 -0
- data/sig/robin/api.rbs +442 -0
- data/sig/robin/batch.rbs +292 -0
- data/sig/robin/cancellation.rbs +25 -0
- data/sig/robin/cli.rbs +102 -0
- data/sig/robin/client.rbs +118 -0
- data/sig/robin/config.rbs +341 -0
- data/sig/robin/cron.rbs +149 -0
- data/sig/robin/debounce.rbs +36 -0
- data/sig/robin/deploy.rbs +24 -0
- data/sig/robin/embedded.rbs +25 -0
- data/sig/robin/encryption.rbs +49 -0
- data/sig/robin/events.rbs +10 -0
- data/sig/robin/expiry.rbs +17 -0
- data/sig/robin/fetcher.rbs +134 -0
- data/sig/robin/gvl.rbs +39 -0
- data/sig/robin/health.rbs +30 -0
- data/sig/robin/iterable_job.rbs +99 -0
- data/sig/robin/job.rbs +173 -0
- data/sig/robin/job_class.rbs +55 -0
- data/sig/robin/job_logger.rbs +56 -0
- data/sig/robin/launcher.rbs +191 -0
- data/sig/robin/leader.rbs +43 -0
- data/sig/robin/limiter.rbs +317 -0
- data/sig/robin/manager.rbs +54 -0
- data/sig/robin/metrics.rbs +122 -0
- data/sig/robin/middleware.rbs +106 -0
- data/sig/robin/opentelemetry.rbs +51 -0
- data/sig/robin/periodic.rbs +81 -0
- data/sig/robin/processor.rbs +107 -0
- data/sig/robin/profiler.rbs +60 -0
- data/sig/robin/progress.rbs +22 -0
- data/sig/robin/rails/app_loader.rbs +10 -0
- data/sig/robin/rails/current_attributes.rbs +44 -0
- data/sig/robin/rails/error_reporter.rbs +21 -0
- data/sig/robin/rails/notifications.rbs +9 -0
- data/sig/robin/rails/transactional_push.rbs +16 -0
- data/sig/robin/rails.rbs +8 -0
- data/sig/robin/recovery.rbs +29 -0
- data/sig/robin/redis/connection.rbs +224 -0
- data/sig/robin/redis/pool.rbs +89 -0
- data/sig/robin/redis/resp.rbs +29 -0
- data/sig/robin/reliable_push.rbs +77 -0
- data/sig/robin/retrier.rbs +72 -0
- data/sig/robin/scheduler.rbs +40 -0
- data/sig/robin/scripts.rbs +36 -0
- data/sig/robin/statsd.rbs +55 -0
- data/sig/robin/swarm.rbs +133 -0
- data/sig/robin/systemd.rbs +13 -0
- data/sig/robin/testing.rbs +80 -0
- data/sig/robin/top.rbs +79 -0
- data/sig/robin/unique.rbs +72 -0
- data/sig/robin/version.rbs +3 -0
- data/sig/robin/warmup.rbs +44 -0
- data/sig/robin/web/api.rbs +196 -0
- data/sig/robin/web/batch_status.rbs +25 -0
- data/sig/robin/web/extension.rbs +106 -0
- data/sig/robin/web/permissions.rbs +14 -0
- data/sig/robin/web/prometheus.rbs +33 -0
- data/sig/robin/web/router.rbs +53 -0
- data/sig/robin/web/static.rbs +42 -0
- data/sig/robin/web.rbs +140 -0
- data/sig/robin.rbs +116 -0
- data/sig/robin_resp.rbs +55 -0
- metadata +294 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Puts jobs held by crashed processes back on their queues.
|
|
3
|
+
module Recovery
|
|
4
|
+
# RECOVER's reply: ["ok" or "dead", raw job] pairs, flattened.
|
|
5
|
+
type recovered = Array[String]
|
|
6
|
+
|
|
7
|
+
# Recovers every dead process's in-flight jobs; returns how many.
|
|
8
|
+
def self.run: (Config config, ?pool: Redis::Pool) -> Integer
|
|
9
|
+
|
|
10
|
+
# Moves all jobs in a process's in-flight keys back to their queues
|
|
11
|
+
# and unregisters the keys; returns how many.
|
|
12
|
+
def self.drain: (Config config, Redis::Connection conn, String key, count: bool) -> Integer
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
def self.recover: (Config config, Redis::Connection conn, Array[[String, String, bool]] dead) -> Array[Integer]
|
|
17
|
+
|
|
18
|
+
def self.forget: (Config config, Redis::Pipeline pipe, String set, String identity) -> void
|
|
19
|
+
|
|
20
|
+
# Runs RECOVER on +redis+ (a connection), or queues it (a pipeline).
|
|
21
|
+
def self.queue_recover: (Config config, Redis::Connection | Redis::Pipeline redis, String key, bool count) -> Redis::reply
|
|
22
|
+
|
|
23
|
+
def self.settle: (Config config, Redis::Connection conn, Array[[recovered, bool?]] results) -> Array[Integer]
|
|
24
|
+
|
|
25
|
+
def self.recovered_job: (Config config, String raw, bool poison) -> void
|
|
26
|
+
|
|
27
|
+
def self.mark_poison: (Config config, Redis::Connection conn, Array[String] raws) -> void
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
module Redis
|
|
3
|
+
# A reply from Redis: a String, Integer, nil, an Array of replies, or
|
|
4
|
+
# (nested in an Array) a CommandError. What a command returns depends on
|
|
5
|
+
# the command, so callers know its shape, not the type checker.
|
|
6
|
+
type reply = untyped
|
|
7
|
+
|
|
8
|
+
# A command: its name and arguments (anything with #to_s).
|
|
9
|
+
type command = Array[untyped]
|
|
10
|
+
|
|
11
|
+
# The reply parser (and command encoder): Robin::Redis::RESP, or the
|
|
12
|
+
# extension's Robin::Redis::NativeRESP.
|
|
13
|
+
interface _Parser
|
|
14
|
+
def parse: (String buf, Integer pos) -> [reply, Integer]?
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
interface _Encoder
|
|
18
|
+
def encode: (command args, ?String out) -> String
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
class ConnectionError < StandardError
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
class TimeoutError < ConnectionError
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Picks the reply parser: the C (or Java) extension's when it's
|
|
28
|
+
# available, else the pure-Ruby one.
|
|
29
|
+
# Loads the extension (Kernel#require).
|
|
30
|
+
interface _Loader
|
|
31
|
+
def call: (String feature) -> boolish
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.parser: (?disabled: bool, ?loader: _Loader) -> (singleton(RESP) | singleton(NativeRESP))
|
|
35
|
+
|
|
36
|
+
PARSER: singleton(RESP) | singleton(NativeRESP)
|
|
37
|
+
|
|
38
|
+
# The extension's socket reader, when the extension's parser is in use.
|
|
39
|
+
def self.native_io: (singleton(RESP) | singleton(NativeRESP) parser, singleton(Redis) mod) -> singleton(NativeIO)?
|
|
40
|
+
|
|
41
|
+
NATIVE_IO: singleton(NativeIO)?
|
|
42
|
+
|
|
43
|
+
# Encodes commands: the extension's encoder where it has one.
|
|
44
|
+
def self.encoder: (singleton(RESP) | singleton(NativeRESP) parser) -> (singleton(RESP) | singleton(NativeRESP))
|
|
45
|
+
|
|
46
|
+
ENCODER: singleton(RESP) | singleton(NativeRESP)
|
|
47
|
+
|
|
48
|
+
# What Connection, Pool and Pipeline share: sending a command.
|
|
49
|
+
interface _Caller
|
|
50
|
+
def call: (*untyped args) -> reply
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# A single, non-thread-safe connection to a Redis server.
|
|
54
|
+
class Connection
|
|
55
|
+
DEFAULTS: Hash[Symbol, untyped]
|
|
56
|
+
|
|
57
|
+
SENTINEL_PORT: Integer
|
|
58
|
+
|
|
59
|
+
SCHEMES: Array[String]
|
|
60
|
+
|
|
61
|
+
TLS_SCHEMES: Array[String]
|
|
62
|
+
|
|
63
|
+
READ_CHUNK: Integer
|
|
64
|
+
|
|
65
|
+
RECONNECT_ERRORS: Array[String]
|
|
66
|
+
|
|
67
|
+
@options: Hash[Symbol, untyped]
|
|
68
|
+
@sentinels: Array[[String, Integer]]?
|
|
69
|
+
@sock: (Socket | UNIXSocket | OpenSSL::SSL::SSLSocket)?
|
|
70
|
+
@buf: String
|
|
71
|
+
@pos: Integer
|
|
72
|
+
@scripts: Set[String]
|
|
73
|
+
@native_io: singleton(NativeIO)?
|
|
74
|
+
|
|
75
|
+
# The options it connects with (see DEFAULTS).
|
|
76
|
+
attr_reader options: Hash[Symbol, untyped]
|
|
77
|
+
|
|
78
|
+
# +url+ with any password (or user) replaced, fit for messages.
|
|
79
|
+
def self.redact: (_ToS url) -> String
|
|
80
|
+
|
|
81
|
+
def self.parse_url: (String url) -> Hash[Symbol, untyped]
|
|
82
|
+
|
|
83
|
+
private def self.tcp_options: (URI::Generic uri) -> Hash[Symbol, untyped]
|
|
84
|
+
|
|
85
|
+
# The Redis URL from the environment.
|
|
86
|
+
def self.env_url: (?_Env env) -> String?
|
|
87
|
+
|
|
88
|
+
# Options: +url+ (redis://, rediss://, valkey://, valkeys:// or unix://)
|
|
89
|
+
# or the parts of one, timeouts in seconds, TLS +ssl_params+ (see
|
|
90
|
+
# OpenSSL::SSL::SSLContext#set_params), and Sentinel's.
|
|
91
|
+
def initialize: (
|
|
92
|
+
?url: String?,
|
|
93
|
+
?host: String,
|
|
94
|
+
?port: Integer,
|
|
95
|
+
?db: Integer,
|
|
96
|
+
?path: String?,
|
|
97
|
+
?username: String?,
|
|
98
|
+
?password: String?,
|
|
99
|
+
?ssl: bool,
|
|
100
|
+
?ssl_params: Hash[Symbol, untyped],
|
|
101
|
+
?connect_timeout: seconds,
|
|
102
|
+
?read_timeout: seconds,
|
|
103
|
+
?write_timeout: seconds,
|
|
104
|
+
?name: String?,
|
|
105
|
+
?sentinels: Array[String | Hash[Symbol | String, String | Integer]]?,
|
|
106
|
+
?master_name: String?,
|
|
107
|
+
?sentinel_username: String?,
|
|
108
|
+
?sentinel_password: String?
|
|
109
|
+
) -> void
|
|
110
|
+
|
|
111
|
+
# Normalizes the +sentinels:+ option to [host, port] pairs.
|
|
112
|
+
def self.sentinel_list: (Array[String | Hash[Symbol | String, String | Integer]]? sentinels) -> Array[[String, Integer]]?
|
|
113
|
+
|
|
114
|
+
def inspect: () -> String
|
|
115
|
+
|
|
116
|
+
# The server it talks to, for messages: "host:port/db", or the socket's path.
|
|
117
|
+
def id: () -> String
|
|
118
|
+
|
|
119
|
+
# Sends a command and returns its reply.
|
|
120
|
+
def call: (*untyped args) -> reply
|
|
121
|
+
|
|
122
|
+
# Like #call, for blocking commands: +timeout+ extends the read timeout.
|
|
123
|
+
def blocking_call: (seconds timeout, *untyped args) -> reply
|
|
124
|
+
|
|
125
|
+
# Sends several commands in one round trip; returns their replies.
|
|
126
|
+
def pipelined: (?timeout: seconds) { (Pipeline pipe) -> void } -> Array[reply]
|
|
127
|
+
|
|
128
|
+
# MULTI/EXEC transaction. Returns the EXEC reply.
|
|
129
|
+
def multi: () { (Pipeline pipe) -> void } -> Array[reply]
|
|
130
|
+
|
|
131
|
+
# Runs a Lua script by SHA, loading it the first time.
|
|
132
|
+
def evalsha: (Script script, ?Array[untyped] keys, ?Array[untyped]? argv) -> reply
|
|
133
|
+
|
|
134
|
+
def close: () -> void
|
|
135
|
+
|
|
136
|
+
# Lets go of the connection without a word to the server (after a fork).
|
|
137
|
+
def abandon: () -> void
|
|
138
|
+
|
|
139
|
+
private
|
|
140
|
+
|
|
141
|
+
def reset: () -> void
|
|
142
|
+
|
|
143
|
+
def call_with_timeout: (seconds read_timeout, command args, ?retried: bool) -> reply
|
|
144
|
+
|
|
145
|
+
def raise_first_error: (Array[reply] results) -> void
|
|
146
|
+
|
|
147
|
+
def reconnect_error?: (CommandError error) -> bool
|
|
148
|
+
|
|
149
|
+
def encode: (Array[command] commands) -> String
|
|
150
|
+
|
|
151
|
+
def roundtrip: (Array[command] commands, seconds read_timeout) -> Array[reply]
|
|
152
|
+
|
|
153
|
+
def send_commands: (String payload, seconds read_timeout) -> void
|
|
154
|
+
|
|
155
|
+
def roundtrip?: () -> bool
|
|
156
|
+
|
|
157
|
+
def send_and_read: (String payload, seconds read_timeout, ?reconnected: bool) -> void
|
|
158
|
+
|
|
159
|
+
def write_failed: ([Integer, Integer?] failure) -> bot
|
|
160
|
+
|
|
161
|
+
def read_replies: (Integer count, seconds read_timeout) -> Array[reply]
|
|
162
|
+
|
|
163
|
+
def stale?: () -> bool
|
|
164
|
+
|
|
165
|
+
def connect: () -> void
|
|
166
|
+
|
|
167
|
+
def open_socket: () -> (Socket | UNIXSocket | OpenSSL::SSL::SSLSocket)
|
|
168
|
+
|
|
169
|
+
# The master's [host, port], as the Sentinel answered.
|
|
170
|
+
def sentinel_master: () -> [String, String]
|
|
171
|
+
|
|
172
|
+
def ask_sentinel: (String host, Integer port, String name) -> ([String, String] | String | ConnectionError | CommandError)
|
|
173
|
+
|
|
174
|
+
def handshake: () -> void
|
|
175
|
+
|
|
176
|
+
def wrap_ssl: (Socket sock, String host) -> OpenSSL::SSL::SSLSocket
|
|
177
|
+
|
|
178
|
+
def write: (String data) -> void
|
|
179
|
+
|
|
180
|
+
def read_reply: (seconds timeout) -> reply
|
|
181
|
+
|
|
182
|
+
def consume: ([reply, Integer] result) -> reply
|
|
183
|
+
|
|
184
|
+
def fill: (seconds timeout) -> void
|
|
185
|
+
|
|
186
|
+
def native_fill: (seconds timeout) -> void
|
|
187
|
+
|
|
188
|
+
def filled: (Integer | false | nil read) -> void
|
|
189
|
+
|
|
190
|
+
# +kind+: :wait_readable or :wait_writable.
|
|
191
|
+
def wait: (Socket | UNIXSocket | OpenSSL::SSL::SSLSocket io, Symbol kind, seconds timeout) -> void
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Collects commands for Connection#pipelined and Connection#multi.
|
|
195
|
+
class Pipeline
|
|
196
|
+
NONE: Array[untyped]
|
|
197
|
+
|
|
198
|
+
@scripts: Set[String]?
|
|
199
|
+
@loads: Hash[Integer, String]?
|
|
200
|
+
|
|
201
|
+
attr_reader commands: Array[command]
|
|
202
|
+
|
|
203
|
+
def initialize: (?Set[String]? scripts) -> void
|
|
204
|
+
|
|
205
|
+
# The positions of the SCRIPT LOADs it added.
|
|
206
|
+
def loads: () -> Array[Integer]
|
|
207
|
+
|
|
208
|
+
# The SHAs those SCRIPT LOADs load.
|
|
209
|
+
def loading: () -> Array[String]
|
|
210
|
+
|
|
211
|
+
def call: (*untyped args) -> nil
|
|
212
|
+
|
|
213
|
+
def evalsha: (Script script, ?Array[untyped] keys, ?Array[untyped]? argv) -> nil
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# A Lua script with a precomputed SHA1 for EVALSHA.
|
|
217
|
+
class Script
|
|
218
|
+
attr_reader source: String
|
|
219
|
+
attr_reader sha: String
|
|
220
|
+
|
|
221
|
+
def initialize: (String source) -> void
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
module Redis
|
|
3
|
+
# A small, thread-safe connection pool. Nested #with calls from the same
|
|
4
|
+
# thread reuse the connection that's already checked out.
|
|
5
|
+
class Pool
|
|
6
|
+
class TimeoutError < Robin::Redis::TimeoutError
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
# The pool's bookkeeping: how many more connections may be checked
|
|
10
|
+
# out, and the idle ones.
|
|
11
|
+
interface _Slots
|
|
12
|
+
# An idle connection, :create (a slot to fill with a new one), or
|
|
13
|
+
# nil after waiting +timeout+ seconds for either.
|
|
14
|
+
def take: (seconds timeout) -> (Connection | :create | nil)
|
|
15
|
+
|
|
16
|
+
# Returns a connection, idle since +since+ (monotonic seconds).
|
|
17
|
+
def put: (Connection conn, seconds since) -> void
|
|
18
|
+
|
|
19
|
+
# Frees a slot taken with :create but never filled.
|
|
20
|
+
def release: () -> void
|
|
21
|
+
|
|
22
|
+
# Makes +count+ more connections available.
|
|
23
|
+
def add: (Integer count) -> void
|
|
24
|
+
|
|
25
|
+
# Removes and returns the connections idle since before +cutoff+.
|
|
26
|
+
def reap: (seconds cutoff) -> Array[Connection]
|
|
27
|
+
|
|
28
|
+
# Removes and returns every idle connection.
|
|
29
|
+
def drain: () -> Array[Connection]
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
class Slots
|
|
33
|
+
include _Slots
|
|
34
|
+
|
|
35
|
+
@mutex: Thread::Mutex
|
|
36
|
+
@cond: Thread::ConditionVariable
|
|
37
|
+
@free: Integer
|
|
38
|
+
@idle: Array[[Connection, seconds]]
|
|
39
|
+
|
|
40
|
+
def initialize: (Integer size) -> void
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# The extension's slots where there are some (JRuby), else Slots.
|
|
44
|
+
def self.slots_class: (?singleton(Redis) redis) -> (singleton(Slots) | singleton(NativeSlots))
|
|
45
|
+
|
|
46
|
+
SLOTS: singleton(Slots) | singleton(NativeSlots)
|
|
47
|
+
|
|
48
|
+
@factory: ^() -> Connection
|
|
49
|
+
@slots: _Slots
|
|
50
|
+
@key: Symbol
|
|
51
|
+
@pid: Integer
|
|
52
|
+
@reset: Thread::Mutex
|
|
53
|
+
|
|
54
|
+
attr_reader size: Integer
|
|
55
|
+
attr_reader timeout: seconds
|
|
56
|
+
attr_reader idle_timeout: seconds?
|
|
57
|
+
|
|
58
|
+
def initialize: (?size: Integer, ?timeout: seconds, ?idle_timeout: seconds?) { () -> Connection } -> void
|
|
59
|
+
|
|
60
|
+
# Checks out a connection for the block (this thread's, if it has one).
|
|
61
|
+
def with: [T] () { (Connection conn) -> T } -> T
|
|
62
|
+
|
|
63
|
+
def call: (*untyped args) -> reply
|
|
64
|
+
|
|
65
|
+
def pipelined: () { (Pipeline pipe) -> void } -> Array[reply]
|
|
66
|
+
|
|
67
|
+
def multi: () { (Pipeline pipe) -> void } -> Array[reply]
|
|
68
|
+
|
|
69
|
+
def evalsha: (Script script, ?Array[untyped] keys, ?Array[untyped] argv) -> reply
|
|
70
|
+
|
|
71
|
+
def blocking_call: (seconds timeout, *untyped args) -> reply
|
|
72
|
+
|
|
73
|
+
def shutdown: () -> void
|
|
74
|
+
|
|
75
|
+
# Grows the pool to +size+ connections; returns the pool's size.
|
|
76
|
+
def grow: (Integer size) -> Integer
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def checkout: () -> Connection
|
|
81
|
+
|
|
82
|
+
def checkin: (Connection conn) -> void
|
|
83
|
+
|
|
84
|
+
def monotonic: () -> Float
|
|
85
|
+
|
|
86
|
+
def reset_after_fork: () -> void
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
module Redis
|
|
3
|
+
# Raised for "-ERR ..." replies (returned, not raised, when nested in an
|
|
4
|
+
# Array reply).
|
|
5
|
+
class CommandError < StandardError
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
class ProtocolError < StandardError
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# Encodes commands and decodes replies in RESP2.
|
|
12
|
+
module RESP
|
|
13
|
+
CRLF: String
|
|
14
|
+
|
|
15
|
+
# Appends the command +args+ (anything responding to #to_s) to +out+.
|
|
16
|
+
def self.encode: (command args, ?String out) -> String
|
|
17
|
+
|
|
18
|
+
# [value, next_offset], or nil while +buf+ doesn't yet hold a whole
|
|
19
|
+
# reply at +pos+.
|
|
20
|
+
def self.parse: (String buf, Integer pos) -> [reply, Integer]?
|
|
21
|
+
|
|
22
|
+
private def self.parse_bulk: (String buf, Integer pos, Integer len) -> [String?, Integer]?
|
|
23
|
+
|
|
24
|
+
private def self.parse_array: (String buf, Integer pos, Integer count) -> [Array[reply]?, Integer]?
|
|
25
|
+
|
|
26
|
+
private def self.utf8: (String str) -> String
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Keeps pushes from failing while Redis is briefly unreachable
|
|
3
|
+
# (Config#reliable_push).
|
|
4
|
+
class ReliablePush
|
|
5
|
+
DEFAULT_LIMIT: Integer
|
|
6
|
+
|
|
7
|
+
DRAIN_CHUNK: Integer
|
|
8
|
+
|
|
9
|
+
ERRORS: Array[singleton(StandardError)]
|
|
10
|
+
|
|
11
|
+
# Jobs dropped because the buffer was full.
|
|
12
|
+
class Dropped < Error
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Jobs still buffered when the process exited.
|
|
16
|
+
class Lost < Error
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# ReliablePush.exit_hook: called with each instance the first time it
|
|
20
|
+
# buffers, to have it #finish when the process exits.
|
|
21
|
+
interface _ExitHook
|
|
22
|
+
def call: (ReliablePush reliable_push) -> void
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
self.@instances: Hash[[Integer, Integer], ReliablePush]
|
|
26
|
+
self.@lock: Thread::Mutex
|
|
27
|
+
self.@exit_hook: _ExitHook
|
|
28
|
+
|
|
29
|
+
AT_EXIT: ^(ReliablePush reliable_push) -> void
|
|
30
|
+
|
|
31
|
+
def self.exit_hook: () -> (_ExitHook)
|
|
32
|
+
|
|
33
|
+
def self.exit_hook=: (_ExitHook hook) -> (_ExitHook)
|
|
34
|
+
|
|
35
|
+
# One buffer per config and Redis pool.
|
|
36
|
+
def self.for: (Config config, ?Redis::Pool? pool) -> ReliablePush
|
|
37
|
+
|
|
38
|
+
@config: Config
|
|
39
|
+
@pool: Redis::Pool?
|
|
40
|
+
@buffer: Array[job_hash]
|
|
41
|
+
@mutex: Thread::Mutex
|
|
42
|
+
@hooked: bool?
|
|
43
|
+
@retrier: Thread?
|
|
44
|
+
|
|
45
|
+
# The most jobs it buffers.
|
|
46
|
+
attr_reader limit: Integer
|
|
47
|
+
|
|
48
|
+
def initialize: (Config config, ?limit: Integer?, ?pool: Redis::Pool?) -> void
|
|
49
|
+
|
|
50
|
+
def size: () -> Integer
|
|
51
|
+
|
|
52
|
+
# Yields to push +payloads+; buffers them if Redis is unreachable.
|
|
53
|
+
def push: (Array[job_hash] payloads) { () -> void } -> void
|
|
54
|
+
|
|
55
|
+
# Pushes buffered jobs, oldest first. True if the buffer is empty.
|
|
56
|
+
def drain: () -> bool
|
|
57
|
+
|
|
58
|
+
# One last try to push what's buffered (at exit).
|
|
59
|
+
def finish: () -> void
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def append: (Array[job_hash] payloads) -> Array[job_hash]
|
|
64
|
+
|
|
65
|
+
def report_dropped: (Array[job_hash] dropped) -> void
|
|
66
|
+
|
|
67
|
+
def report_pushed: (Integer count) -> void
|
|
68
|
+
|
|
69
|
+
def push_chunk: (Array[job_hash] chunk) -> void
|
|
70
|
+
|
|
71
|
+
def describe: (Array[job_hash] jobs) -> String
|
|
72
|
+
|
|
73
|
+
def start_retrier: () -> void
|
|
74
|
+
|
|
75
|
+
def pause: (seconds seconds) -> void
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Decides what happens to a job that raised: schedule a retry with
|
|
3
|
+
# exponential backoff, move it to the dead set, or discard it.
|
|
4
|
+
class Retrier
|
|
5
|
+
# What #handle did with a failed job.
|
|
6
|
+
type outcome = :retry | :dead | :discard
|
|
7
|
+
|
|
8
|
+
MAX_MESSAGE_BYTES: Integer
|
|
9
|
+
|
|
10
|
+
MAX_DELAY: Integer
|
|
11
|
+
|
|
12
|
+
BACKTRACE_PREFIX: String
|
|
13
|
+
|
|
14
|
+
# Fields a failure adds to a job before it's written back to Redis.
|
|
15
|
+
FAILURE_FIELDS: Array[String]
|
|
16
|
+
|
|
17
|
+
# Backtrace lines, compressed (base64 of deflated JSON); [] stays [].
|
|
18
|
+
def self.pack_backtrace: (Array[String] lines) -> (String | Array[String])
|
|
19
|
+
|
|
20
|
+
# A stored error_backtrace (compressed, or a plain array) as lines.
|
|
21
|
+
def self.unpack_backtrace: (String | Array[String] | nil value) -> Array[String]?
|
|
22
|
+
|
|
23
|
+
# Default backoff (seconds) before retry number +failures+.
|
|
24
|
+
def self.default_delay: (Integer failures) -> Integer
|
|
25
|
+
|
|
26
|
+
# A custom delay pushed up to 10% (at most 10s) later.
|
|
27
|
+
def self.jitter: (Numeric seconds) -> Numeric
|
|
28
|
+
|
|
29
|
+
# An exception's message as valid UTF-8, even when reading it raises.
|
|
30
|
+
def self.message_of: (Exception exception) -> String
|
|
31
|
+
|
|
32
|
+
attr_reader config: Config
|
|
33
|
+
|
|
34
|
+
def initialize: (Config config) -> void
|
|
35
|
+
|
|
36
|
+
# How many times the job may be retried.
|
|
37
|
+
def max_retries: (job_hash payload) -> (Integer | Float)
|
|
38
|
+
|
|
39
|
+
# Records the failure and acknowledges the unit of work.
|
|
40
|
+
def handle: (Fetcher::UnitOfWork unit, job_hash payload, Class job_class, Exception exception, ?batch_failure: Batch::error_info?) -> outcome
|
|
41
|
+
|
|
42
|
+
# Moves a job to the dead set (or discards it, if its hook asks).
|
|
43
|
+
def die: (Fetcher::UnitOfWork? unit, job_hash payload, Class job_class, Exception exception) -> (:dead | :discard)
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def discard: (Fetcher::UnitOfWork? unit, job_hash payload, Exception exception) -> :discard
|
|
48
|
+
|
|
49
|
+
def terminal: (job_hash payload, Exception exception) { (Redis::Pipeline tx) -> void } -> void
|
|
50
|
+
|
|
51
|
+
def discard_requested?: (Class job_class, job_hash payload, Exception exception) -> bool
|
|
52
|
+
|
|
53
|
+
def stored: (job_hash payload) -> job_hash
|
|
54
|
+
|
|
55
|
+
def out_of_time?: (job_hash payload, Integer at) -> boolish
|
|
56
|
+
|
|
57
|
+
def retry_delay: (Class job_class, Integer failures, Exception exception, job_hash payload) -> (Numeric | :discard | :kill)
|
|
58
|
+
|
|
59
|
+
# The class whose retry hooks apply (one that may be a Robin::Job).
|
|
60
|
+
def hook_class: (Class job_class, job_hash payload) -> untyped
|
|
61
|
+
|
|
62
|
+
def custom_delay: (^(Integer, Exception, job_hash) -> untyped block, Integer failures, Exception exception, job_hash payload) -> (Numeric | :discard | :kill | nil)
|
|
63
|
+
|
|
64
|
+
def backtrace_of: (Exception exception, Integer lines) -> Array[String]
|
|
65
|
+
|
|
66
|
+
def clean: (Array[String]? backtrace) -> Array[String]?
|
|
67
|
+
|
|
68
|
+
def backtrace_lines: (job_hash payload) -> Integer?
|
|
69
|
+
|
|
70
|
+
def truncate: (String message) -> String
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Moves due jobs from the schedule and retry sets onto their queues.
|
|
3
|
+
class Scheduler
|
|
4
|
+
BATCH: Integer
|
|
5
|
+
|
|
6
|
+
SETS: Array[String]
|
|
7
|
+
|
|
8
|
+
MIN_WAIT: Float
|
|
9
|
+
|
|
10
|
+
@config: Config
|
|
11
|
+
@mutex: Thread::Mutex
|
|
12
|
+
@cond: Thread::ConditionVariable
|
|
13
|
+
@next: Hash[String, Float?]
|
|
14
|
+
# nil until #start
|
|
15
|
+
@thread: Thread
|
|
16
|
+
@done: bool?
|
|
17
|
+
@processes: Integer?
|
|
18
|
+
|
|
19
|
+
def initialize: (Config config) -> void
|
|
20
|
+
|
|
21
|
+
def start: () -> self
|
|
22
|
+
|
|
23
|
+
def stop: () -> Thread?
|
|
24
|
+
|
|
25
|
+
# Enqueues all currently due jobs. Returns the number moved.
|
|
26
|
+
def enqueue_due: (?Integer now) -> Integer
|
|
27
|
+
|
|
28
|
+
# When the next job of either set is due (epoch seconds), as of the
|
|
29
|
+
# last poll; nil if both were empty.
|
|
30
|
+
def next_due: () -> Float?
|
|
31
|
+
|
|
32
|
+
# Seconds until the next poll (for the leader, until the next job is
|
|
33
|
+
# due, if sooner).
|
|
34
|
+
def interval: () -> seconds
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def sleep_for: (seconds seconds) -> void
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# The Lua scripts Robin runs in Redis.
|
|
3
|
+
module Scripts
|
|
4
|
+
FETCH: Redis::Script
|
|
5
|
+
|
|
6
|
+
ENQUEUE_DUE: Redis::Script
|
|
7
|
+
|
|
8
|
+
RECOVER: Redis::Script
|
|
9
|
+
|
|
10
|
+
FORGET: Redis::Script
|
|
11
|
+
|
|
12
|
+
PRUNE: Redis::Script
|
|
13
|
+
|
|
14
|
+
QUEUE_SAMPLES: Redis::Script
|
|
15
|
+
|
|
16
|
+
REQUEUE_ONE: Redis::Script
|
|
17
|
+
|
|
18
|
+
LEASE: Redis::Script
|
|
19
|
+
|
|
20
|
+
RELEASE: Redis::Script
|
|
21
|
+
|
|
22
|
+
FIND_JID: Redis::Script
|
|
23
|
+
|
|
24
|
+
DELETE_CLASS: Redis::Script
|
|
25
|
+
|
|
26
|
+
STATS: Redis::Script
|
|
27
|
+
|
|
28
|
+
MEMBER_HASHES: Redis::Script
|
|
29
|
+
|
|
30
|
+
# The hash of each member of +set+, in one round trip: [member, hash]
|
|
31
|
+
# pairs, by member.
|
|
32
|
+
def self.member_hashes: (Config config, String set, String prefix, ?String suffix) -> Array[[String, Hash[String, String]]]
|
|
33
|
+
|
|
34
|
+
HMAX: Redis::Script
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
module Robin
|
|
2
|
+
# Sends job and queue metrics to StatsD or DogStatsD over UDP.
|
|
3
|
+
class Statsd
|
|
4
|
+
# Tags: a value (anything with #to_s) or an Array of them, which
|
|
5
|
+
# repeats the tag once per element (DogStatsD).
|
|
6
|
+
type tags = Hash[Symbol | String, Object]
|
|
7
|
+
|
|
8
|
+
MAX_PACKET: Integer
|
|
9
|
+
|
|
10
|
+
@dogstatsd: bool
|
|
11
|
+
@distributions: bool
|
|
12
|
+
@mutex: Thread::Mutex
|
|
13
|
+
@socket: UDPSocket?
|
|
14
|
+
|
|
15
|
+
attr_reader host: String
|
|
16
|
+
attr_reader port: Integer
|
|
17
|
+
|
|
18
|
+
# "<prefix>." (or "" for none).
|
|
19
|
+
attr_reader prefix: String
|
|
20
|
+
|
|
21
|
+
# Tags sent with every metric (DogStatsD).
|
|
22
|
+
attr_reader tags: tags
|
|
23
|
+
|
|
24
|
+
def initialize: (?host: String, ?port: Integer, ?prefix: _ToS?, ?tags: tags, ?dogstatsd: bool, ?distributions: bool) -> void
|
|
25
|
+
|
|
26
|
+
def dogstatsd?: () -> bool
|
|
27
|
+
|
|
28
|
+
def distributions?: () -> bool
|
|
29
|
+
|
|
30
|
+
def increment: (String name, ?Numeric value, ?tags: tags) -> nil
|
|
31
|
+
|
|
32
|
+
def gauge: (String name, Numeric value, ?tags: tags) -> nil
|
|
33
|
+
|
|
34
|
+
def timing: (String name, Integer | Float ms, ?tags: tags) -> nil
|
|
35
|
+
|
|
36
|
+
# A Datadog distribution sample (needs dogstatsd: true).
|
|
37
|
+
def distribution: (String name, Integer | Float value, ?tags: tags) -> nil
|
|
38
|
+
|
|
39
|
+
# +jobs+: per-class counts as Metrics::Collector drains them.
|
|
40
|
+
def job_metrics: (Metrics::jobs jobs) -> nil
|
|
41
|
+
|
|
42
|
+
# +queues+: {name => {size:, latency:}}.
|
|
43
|
+
def queue_metrics: (Hash[String, Metrics::queue_sample] queues) -> nil
|
|
44
|
+
|
|
45
|
+
def close: () -> nil
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def line: (String name, Numeric value, String type, tags tags) -> String
|
|
50
|
+
|
|
51
|
+
def send_lines: (Array[String] lines) -> nil
|
|
52
|
+
|
|
53
|
+
def deliver: (String packet) -> void
|
|
54
|
+
end
|
|
55
|
+
end
|