robin 0.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/java/robin_resp/Blocking.java +27 -0
  13. data/ext/java/robin_resp/NativeIO.java +229 -0
  14. data/ext/java/robin_resp/NativeRESP.java +279 -0
  15. data/ext/java/robin_resp/NativeSlots.java +133 -0
  16. data/ext/java/robin_resp/RobinRespService.java +32 -0
  17. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  18. data/lib/generators/robin/install/install_generator.rb +39 -0
  19. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  20. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  21. data/lib/generators/robin/job/job_generator.rb +78 -0
  22. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  23. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  24. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  25. data/lib/puma/plugin/robin.rb +131 -0
  26. data/lib/robin/active_job.rb +144 -0
  27. data/lib/robin/api.rb +777 -0
  28. data/lib/robin/batch.rb +700 -0
  29. data/lib/robin/cancellation.rb +62 -0
  30. data/lib/robin/cli.rb +325 -0
  31. data/lib/robin/client.rb +372 -0
  32. data/lib/robin/config.rb +461 -0
  33. data/lib/robin/cron.rb +325 -0
  34. data/lib/robin/debounce.rb +108 -0
  35. data/lib/robin/deploy.rb +65 -0
  36. data/lib/robin/embedded.rb +53 -0
  37. data/lib/robin/encryption.rb +147 -0
  38. data/lib/robin/events.rb +60 -0
  39. data/lib/robin/expiry.rb +41 -0
  40. data/lib/robin/fetcher.rb +441 -0
  41. data/lib/robin/gvl.rb +62 -0
  42. data/lib/robin/health.rb +76 -0
  43. data/lib/robin/iterable_job.rb +294 -0
  44. data/lib/robin/job.rb +258 -0
  45. data/lib/robin/job_class.rb +157 -0
  46. data/lib/robin/job_logger.rb +121 -0
  47. data/lib/robin/launcher.rb +658 -0
  48. data/lib/robin/leader.rb +86 -0
  49. data/lib/robin/limiter.rb +639 -0
  50. data/lib/robin/manager.rb +125 -0
  51. data/lib/robin/metrics.rb +303 -0
  52. data/lib/robin/middleware.rb +165 -0
  53. data/lib/robin/opentelemetry.rb +88 -0
  54. data/lib/robin/periodic.rb +327 -0
  55. data/lib/robin/processor.rb +317 -0
  56. data/lib/robin/profiler.rb +140 -0
  57. data/lib/robin/progress.rb +46 -0
  58. data/lib/robin/rails/app_loader.rb +23 -0
  59. data/lib/robin/rails/current_attributes.rb +88 -0
  60. data/lib/robin/rails/error_reporter.rb +31 -0
  61. data/lib/robin/rails/notifications.rb +22 -0
  62. data/lib/robin/rails/railtie.rb +87 -0
  63. data/lib/robin/rails/transactional_push.rb +49 -0
  64. data/lib/robin/rails.rb +23 -0
  65. data/lib/robin/recovery.rb +135 -0
  66. data/lib/robin/redis/connection.rb +639 -0
  67. data/lib/robin/redis/pool.rb +190 -0
  68. data/lib/robin/redis/resp.rb +77 -0
  69. data/lib/robin/reliable_push.rb +153 -0
  70. data/lib/robin/retrier.rb +284 -0
  71. data/lib/robin/scheduler.rb +102 -0
  72. data/lib/robin/scripts.rb +331 -0
  73. data/lib/robin/statsd.rb +135 -0
  74. data/lib/robin/swarm.rb +319 -0
  75. data/lib/robin/systemd.rb +47 -0
  76. data/lib/robin/testing.rb +158 -0
  77. data/lib/robin/top.rb +178 -0
  78. data/lib/robin/unique.rb +198 -0
  79. data/lib/robin/version.rb +5 -0
  80. data/lib/robin/warmup.rb +94 -0
  81. data/lib/robin/web/api.rb +580 -0
  82. data/lib/robin/web/batch_status.rb +44 -0
  83. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  84. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  86. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  87. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  88. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  89. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  90. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  91. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  92. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  93. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  94. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  95. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  96. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  97. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  98. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  99. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  100. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  101. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  102. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  103. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  104. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  105. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  106. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  107. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  108. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  109. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  110. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  111. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  112. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  113. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  114. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  115. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  116. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  117. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  118. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  119. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  120. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  121. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  122. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  123. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  124. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  125. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  126. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  127. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  128. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  129. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  130. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  131. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  132. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  133. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  134. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  135. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  136. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  137. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  138. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  139. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  140. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  141. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  142. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  143. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  144. data/lib/robin/web/dist/index.html +19 -0
  145. data/lib/robin/web/extension.rb +193 -0
  146. data/lib/robin/web/permissions.rb +40 -0
  147. data/lib/robin/web/prometheus.rb +79 -0
  148. data/lib/robin/web/router.rb +150 -0
  149. data/lib/robin/web/static.rb +110 -0
  150. data/lib/robin/web.rb +196 -0
  151. data/lib/robin.rb +261 -0
  152. data/lib/robin_resp/robin_resp.jar +0 -0
  153. data/sig/manifest.yaml +9 -0
  154. data/sig/robin/api.rbs +442 -0
  155. data/sig/robin/batch.rbs +292 -0
  156. data/sig/robin/cancellation.rbs +25 -0
  157. data/sig/robin/cli.rbs +102 -0
  158. data/sig/robin/client.rbs +118 -0
  159. data/sig/robin/config.rbs +341 -0
  160. data/sig/robin/cron.rbs +149 -0
  161. data/sig/robin/debounce.rbs +36 -0
  162. data/sig/robin/deploy.rbs +24 -0
  163. data/sig/robin/embedded.rbs +25 -0
  164. data/sig/robin/encryption.rbs +49 -0
  165. data/sig/robin/events.rbs +10 -0
  166. data/sig/robin/expiry.rbs +17 -0
  167. data/sig/robin/fetcher.rbs +134 -0
  168. data/sig/robin/gvl.rbs +39 -0
  169. data/sig/robin/health.rbs +30 -0
  170. data/sig/robin/iterable_job.rbs +99 -0
  171. data/sig/robin/job.rbs +173 -0
  172. data/sig/robin/job_class.rbs +55 -0
  173. data/sig/robin/job_logger.rbs +56 -0
  174. data/sig/robin/launcher.rbs +191 -0
  175. data/sig/robin/leader.rbs +43 -0
  176. data/sig/robin/limiter.rbs +317 -0
  177. data/sig/robin/manager.rbs +54 -0
  178. data/sig/robin/metrics.rbs +122 -0
  179. data/sig/robin/middleware.rbs +106 -0
  180. data/sig/robin/opentelemetry.rbs +51 -0
  181. data/sig/robin/periodic.rbs +81 -0
  182. data/sig/robin/processor.rbs +107 -0
  183. data/sig/robin/profiler.rbs +60 -0
  184. data/sig/robin/progress.rbs +22 -0
  185. data/sig/robin/rails/app_loader.rbs +10 -0
  186. data/sig/robin/rails/current_attributes.rbs +44 -0
  187. data/sig/robin/rails/error_reporter.rbs +21 -0
  188. data/sig/robin/rails/notifications.rbs +9 -0
  189. data/sig/robin/rails/transactional_push.rbs +16 -0
  190. data/sig/robin/rails.rbs +8 -0
  191. data/sig/robin/recovery.rbs +29 -0
  192. data/sig/robin/redis/connection.rbs +224 -0
  193. data/sig/robin/redis/pool.rbs +89 -0
  194. data/sig/robin/redis/resp.rbs +29 -0
  195. data/sig/robin/reliable_push.rbs +77 -0
  196. data/sig/robin/retrier.rbs +72 -0
  197. data/sig/robin/scheduler.rbs +40 -0
  198. data/sig/robin/scripts.rbs +36 -0
  199. data/sig/robin/statsd.rbs +55 -0
  200. data/sig/robin/swarm.rbs +133 -0
  201. data/sig/robin/systemd.rbs +13 -0
  202. data/sig/robin/testing.rbs +80 -0
  203. data/sig/robin/top.rbs +79 -0
  204. data/sig/robin/unique.rbs +72 -0
  205. data/sig/robin/version.rbs +3 -0
  206. data/sig/robin/warmup.rbs +44 -0
  207. data/sig/robin/web/api.rbs +196 -0
  208. data/sig/robin/web/batch_status.rbs +25 -0
  209. data/sig/robin/web/extension.rbs +106 -0
  210. data/sig/robin/web/permissions.rbs +14 -0
  211. data/sig/robin/web/prometheus.rbs +33 -0
  212. data/sig/robin/web/router.rbs +53 -0
  213. data/sig/robin/web/static.rbs +42 -0
  214. data/sig/robin/web.rbs +140 -0
  215. data/sig/robin.rbs +116 -0
  216. data/sig/robin_resp.rbs +55 -0
  217. metadata +294 -0
@@ -0,0 +1,639 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+ require "uri"
5
+ require "digest/sha1"
6
+ require "openssl"
7
+ require_relative "resp"
8
+
9
+ module Robin
10
+ module Redis
11
+ class ConnectionError < StandardError; end
12
+
13
+ class TimeoutError < ConnectionError; end
14
+
15
+ # Picks the reply parser: the optional C extension when it's available
16
+ # (and not disabled with ROBIN_NO_EXT=1), else the pure-Ruby one.
17
+ def self.parser(disabled: !ENV.fetch("ROBIN_NO_EXT", "").empty?, loader: Kernel.public_method(:require))
18
+ return RESP if disabled
19
+
20
+ loader.call("robin_resp/robin_resp")
21
+ NativeRESP
22
+ rescue LoadError
23
+ RESP
24
+ end
25
+
26
+ PARSER = parser
27
+
28
+ # The C extension's socket reader (Robin::Redis::NativeIO.fill), when the
29
+ # C parser is in use (it's missing where the system has no poll(2)).
30
+ def self.native_io(parser, mod)
31
+ mod::NativeIO if !parser.equal?(RESP) && mod.const_defined?(:NativeIO)
32
+ end
33
+
34
+ NATIVE_IO = native_io(PARSER, self)
35
+
36
+ # Encodes commands: the extension's encoder where it has one (the Java
37
+ # one does), else the pure-Ruby one.
38
+ def self.encoder(parser) = parser.respond_to?(:encode) ? parser : RESP
39
+
40
+ ENCODER = encoder(PARSER)
41
+
42
+ # A single, non-thread-safe connection to a Redis server. Use
43
+ # Robin::Redis::Pool to share connections between threads.
44
+ class Connection
45
+ DEFAULTS = {
46
+ host: "127.0.0.1", port: 6379, db: 0, path: nil, username: nil, password: nil,
47
+ ssl: false, ssl_params: {}, connect_timeout: 1.0, read_timeout: 3.0,
48
+ write_timeout: 3.0, name: nil, sentinels: nil, master_name: nil,
49
+ sentinel_username: nil, sentinel_password: nil
50
+ }.freeze
51
+
52
+ SENTINEL_PORT = 26_379
53
+
54
+ SCHEMES = %w[redis rediss valkey valkeys].freeze
55
+ TLS_SCHEMES = %w[rediss valkeys].freeze
56
+ READ_CHUNK = 65_536
57
+
58
+ # Error replies saying this server can't serve the command right now
59
+ # because of its role or state: a primary demoted to replica by a
60
+ # failover (READONLY, UNBLOCKED), a primary without enough replicas
61
+ # (NOREPLICAS), a replica cut off from its primary (MASTERDOWN), or a
62
+ # server still loading its data (LOADING). None of them ran the
63
+ # command. The connection is closed so the next command reconnects,
64
+ # which reaches the new primary when a proxy, DNS name or VIP has
65
+ # moved; a single command is retried once on the fresh connection.
66
+ # Pipelines and transactions are not retried.
67
+ RECONNECT_ERRORS = %w[READONLY NOREPLICAS UNBLOCKED MASTERDOWN LOADING].freeze
68
+
69
+ attr_reader :options
70
+
71
+ # +url+ with any password (or user) replaced, fit for messages.
72
+ # Everything up to the last "@" goes, as a password may itself
73
+ # contain "@" or "/".
74
+ def self.redact(url) = url.to_s.sub(%r{//.+@}m, "//REDACTED@")
75
+
76
+ def self.parse_url(url)
77
+ uri = begin
78
+ URI.parse(url)
79
+ rescue URI::InvalidURIError
80
+ # Ruby's message would show the URL, password and all.
81
+ raise ArgumentError, "invalid Redis URL: #{redact(url)}"
82
+ end
83
+ opts = if uri.scheme == "unix"
84
+ {path: uri.path}
85
+ elsif SCHEMES.include?(uri.scheme)
86
+ tcp_options(uri)
87
+ else
88
+ raise ArgumentError, "unsupported Redis URL scheme: #{uri.scheme.inspect}"
89
+ end
90
+ user = uri.user.to_s
91
+ opts[:username] = URI.decode_www_form_component(user) unless user.empty?
92
+ if (password = uri.password) then opts[:password] = URI.decode_www_form_component(password) end
93
+ URI.decode_www_form(uri.query.to_s).each { |key, value| opts[:db] = Integer(value) if key == "db" }
94
+ opts
95
+ end
96
+
97
+ def self.tcp_options(uri)
98
+ opts = {ssl: TLS_SCHEMES.include?(uri.scheme)} #: Hash[Symbol, untyped]
99
+ host = uri.host.to_s
100
+ opts[:host] = host unless host.empty?
101
+ opts[:port] = uri.port if uri.port
102
+ db = uri.path.to_s.delete_prefix("/")
103
+ opts[:db] = Integer(db) unless db.empty?
104
+ opts
105
+ end
106
+ private_class_method :tcp_options
107
+
108
+ # The Redis URL from the environment: ROBIN_REDIS_URL, else the
109
+ # variable REDIS_PROVIDER names (e.g. REDIS_PROVIDER=REDISCLOUD_URL, as
110
+ # some hosts' add-ons set), else REDIS_URL.
111
+ def self.env_url(env = ENV)
112
+ provider = env["REDIS_PROVIDER"].to_s
113
+ from_provider = provider.include?("://") ? provider : env[provider]
114
+ env["ROBIN_REDIS_URL"] || from_provider || env["REDIS_URL"]
115
+ end
116
+
117
+ def initialize(**options)
118
+ # @type var options: Hash[Symbol, untyped]
119
+ url = options.delete(:url) || (self.class.env_url unless options.key?(:host) || options.key?(:path))
120
+ options = self.class.parse_url(url).merge(options) if url
121
+ unknown = options.keys - DEFAULTS.keys
122
+ raise ArgumentError, "unknown Redis options: #{unknown.join(", ")}" unless unknown.empty?
123
+
124
+ @options = DEFAULTS.merge(options)
125
+ @sentinels = self.class.sentinel_list(@options.fetch(:sentinels))
126
+ if @sentinels
127
+ @options[:master_name] ||= options[:host]
128
+ raise ArgumentError, "Sentinel needs a master_name (or the URL's host)" unless @options.fetch(:master_name)
129
+ end
130
+ reset
131
+ end
132
+
133
+ # Normalizes the +sentinels:+ option: "host:port" strings or
134
+ # {host:, port:} hashes (the port defaults to 26379).
135
+ # (Each entry becomes a [host, port] pair, which Steep sees as an Array.)
136
+ def self.sentinel_list(sentinels) # steep:ignore MethodBodyTypeMismatch
137
+ return unless sentinels
138
+
139
+ list = Array(sentinels).map do |entry|
140
+ case entry
141
+ when Hash
142
+ entry = entry.transform_keys(&:to_sym)
143
+ [entry.fetch(:host), Integer(entry.fetch(:port, SENTINEL_PORT))]
144
+ else
145
+ host, port = entry.split(":")
146
+ [host, Integer(port || SENTINEL_PORT)]
147
+ end
148
+ end
149
+ raise ArgumentError, "sentinels must not be empty" if list.empty?
150
+
151
+ list
152
+ end
153
+
154
+ # Never shows the password.
155
+ def inspect = "#<#{self.class} #{id}>"
156
+
157
+ def id
158
+ return @options.fetch(:path) if @options.fetch(:path)
159
+
160
+ target = @sentinels ? "sentinel:#{@options.fetch(:master_name)}" : "#{@options.fetch(:host)}:#{@options.fetch(:port)}"
161
+ "#{target}/#{@options.fetch(:db)}"
162
+ end
163
+
164
+ # Sends a command and returns its reply. Error replies are raised as
165
+ # Robin::Redis::CommandError (see RECONNECT_ERRORS for the ones that
166
+ # are retried first).
167
+ def call(*args) = call_with_timeout(@options.fetch(:read_timeout), args)
168
+
169
+ # Like #call, but for blocking commands (BLMOVE, BLPOP...). The read
170
+ # timeout is extended by +timeout+ seconds so the server gets the
171
+ # chance to answer first.
172
+ def blocking_call(timeout, *args) = call_with_timeout(@options.fetch(:read_timeout) + timeout, args)
173
+
174
+ # Sends several commands in one round-trip. Yields a Pipeline; returns
175
+ # the array of replies. The first error reply is raised after all
176
+ # replies have been read. A pipeline holding a blocking command passes
177
+ # its +timeout+, which extends the read timeout like #blocking_call's
178
+ # (Redis runs the commands after it once it returns).
179
+ #
180
+ # A pipeline runs scripts by their SHA once this connection's server
181
+ # has them (see Pipeline#evalsha): it loads each the first time,
182
+ # with a SCRIPT LOAD whose reply isn't among the results.
183
+ def pipelined(timeout: 0)
184
+ pipe = Pipeline.new(@scripts)
185
+ yield pipe
186
+ return [] if pipe.commands.empty?
187
+
188
+ results = roundtrip(pipe.commands, @options.fetch(:read_timeout) + timeout)
189
+ loads = pipe.loads
190
+ unless loads.empty?
191
+ @scripts.merge(pipe.loading)
192
+ results = results.reject.with_index { |_, i| loads.include?(i) }
193
+ end
194
+ raise_first_error(results)
195
+ results
196
+ end
197
+
198
+ # MULTI/EXEC transaction. Returns the EXEC reply (array of results).
199
+ def multi
200
+ pipe = Pipeline.new
201
+ yield pipe
202
+ results = roundtrip([["MULTI"], *pipe.commands, ["EXEC"]], @options.fetch(:read_timeout))
203
+ raise_first_error(results) # errors queueing commands
204
+ exec = results.last
205
+ raise_first_error(exec) # errors running them
206
+ exec
207
+ end
208
+
209
+ # Runs a Lua script by SHA, loading it transparently the first time
210
+ # (in one more round trip, with the script run again).
211
+ def evalsha(script, keys = [], argv = nil)
212
+ result = begin
213
+ # (A nil argv splats to nothing.)
214
+ call("EVALSHA", script.sha, keys.size, *keys, *argv) # steep:ignore ArgumentTypeMismatch
215
+ rescue CommandError => e
216
+ raise unless e.message.start_with?("NOSCRIPT")
217
+
218
+ pipelined do |p|
219
+ p.call("SCRIPT", "LOAD", script.source)
220
+ p.call("EVALSHA", script.sha, keys.size, *keys, *argv) # steep:ignore ArgumentTypeMismatch
221
+ end.last
222
+ end
223
+ @scripts << script.sha # it has it now
224
+ result
225
+ end
226
+
227
+ def close
228
+ @sock&.close
229
+ rescue IOError, SystemCallError
230
+ nil
231
+ ensure
232
+ reset
233
+ end
234
+
235
+ # Lets go of the connection without a word to the server, as a forked
236
+ # child must with the ones it inherited: closing a TLS socket sends a
237
+ # goodbye (close_notify) down the socket the parent still uses, and
238
+ # Redis hangs up on the parent. Closing just this process's file
239
+ # descriptor leaves the parent's connection as it was.
240
+ def abandon
241
+ # (An SSLSocket's #io is the socket under it.)
242
+ (@sock.respond_to?(:io) ? @sock.io : @sock)&.close # steep:ignore NoMethod
243
+ rescue IOError, SystemCallError
244
+ nil
245
+ ensure
246
+ reset
247
+ end
248
+
249
+ private
250
+
251
+ # Forgets the socket and any buffered reply bytes.
252
+ def reset
253
+ @sock = nil
254
+ @buf = "".b
255
+ @pos = 0
256
+ # The SHAs of the scripts the server has, as far as we know (a new
257
+ # socket may reach another server).
258
+ @scripts = Set.new
259
+ end
260
+
261
+ def call_with_timeout(read_timeout, args, retried: false)
262
+ result = roundtrip([args], read_timeout).first
263
+ return result unless result.instance_of?(CommandError)
264
+ raise result unless reconnect_error?(result)
265
+
266
+ close
267
+ raise result if retried
268
+
269
+ call_with_timeout(read_timeout, args, retried: true)
270
+ end
271
+
272
+ def raise_first_error(results)
273
+ error = results.find { |r| r.instance_of?(CommandError) }
274
+ return unless error
275
+
276
+ # The server lost its scripts (SCRIPT FLUSH): load them again.
277
+ @scripts.clear if error.message.start_with?("NOSCRIPT")
278
+ close if reconnect_error?(error)
279
+ raise error
280
+ end
281
+
282
+ def reconnect_error?(error)
283
+ RECONNECT_ERRORS.include?(error.message[/\A\S+/])
284
+ end
285
+
286
+ def encode(commands)
287
+ commands.each_with_object("".b) { |cmd, out| ENCODER.encode(cmd, out) }
288
+ end
289
+
290
+ def roundtrip(commands, read_timeout)
291
+ send_commands(encode(commands), read_timeout)
292
+ read_replies(commands.size, read_timeout)
293
+ end
294
+
295
+ def send_commands(payload, read_timeout)
296
+ retried = false
297
+ begin
298
+ # NativeIO.roundtrip checks the socket itself.
299
+ close if @sock && (roundtrip? ? @pos < @buf.bytesize : stale?)
300
+ connect unless @sock
301
+ roundtrip? ? send_and_read(payload, read_timeout) : write(payload)
302
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, IOError => e
303
+ # The write failed before sending anything (see #write), so the
304
+ # server never saw the command: reconnect and retry once.
305
+ close
306
+ raise ConnectionError, "#{e.class}: #{e} (#{id})" if retried
307
+
308
+ retried = true
309
+ retry
310
+ end
311
+ end
312
+
313
+ def roundtrip? = @native_io.respond_to?(:roundtrip)
314
+
315
+ # With the C extension: checks the connection, sends +payload+, and
316
+ # waits for and reads the start of the replies, in one call (one
317
+ # release of the GVL). Reconnects once if the connection was dropped;
318
+ # a write that fails before sending anything raises its Errno, for
319
+ # #send_commands to retry.
320
+ def send_and_read(payload, read_timeout, reconnected: false)
321
+ # @type ivar @native_io: singleton(NativeIO)
322
+ # @type ivar @sock: Socket | UNIXSocket
323
+ result = begin
324
+ @native_io.roundtrip(@sock, payload, @buf, @options.fetch(:write_timeout), read_timeout)
325
+ rescue SystemCallError => e
326
+ # Only reads raise these: the commands went out, so never retry.
327
+ raise ConnectionError, "#{e.class}: #{e} (#{id})"
328
+ end
329
+ case result
330
+ when :stale
331
+ raise ConnectionError, "connection unusable right after connecting (#{id})" if reconnected
332
+
333
+ close
334
+ connect
335
+ send_and_read(payload, read_timeout, reconnected: true)
336
+ when Array then write_failed(result)
337
+ else filled(result)
338
+ end
339
+ rescue Exception # rubocop:disable Lint/RescueException
340
+ # Anything else (an error, Thread#raise) leaves the connection in
341
+ # an unknown state.
342
+ close
343
+ raise
344
+ end
345
+
346
+ # Raises for a write NativeIO.roundtrip couldn't finish: [bytes written,
347
+ # errno].
348
+ def write_failed(failure)
349
+ written, errno = failure
350
+ raise TimeoutError, "timed out talking to Redis (#{id})" unless errno
351
+
352
+ error = SystemCallError.new("writing to Redis", errno)
353
+ raise error if written.zero?
354
+
355
+ raise ConnectionError, "#{error.class}: #{error} after a partial write (#{id})"
356
+ end
357
+
358
+ def read_replies(count, read_timeout)
359
+ # @type ivar @sock: Socket | UNIXSocket | OpenSSL::SSL::SSLSocket
360
+ completed = false
361
+ # The commands were only just sent, so their replies can't have
362
+ # arrived: wait for them instead of first trying a read (a syscall
363
+ # that would only say "not yet"). Over TLS, OpenSSL may hold bytes
364
+ # the socket doesn't show, so a read comes first there.
365
+ wait(@sock, :wait_readable, read_timeout) unless @native_io || @pos < @buf.bytesize || @options.fetch(:ssl)
366
+ replies = Array.new(count) { read_reply(read_timeout) }
367
+ completed = true
368
+ replies
369
+ rescue SystemCallError, IOError => e
370
+ raise ConnectionError, "#{e.class}: #{e} (#{id})"
371
+ ensure
372
+ # Anything that interrupts us mid-read (an error, a timeout, or
373
+ # Thread#raise) leaves unread replies on the socket; the
374
+ # connection can't be reused.
375
+ close unless completed
376
+ end
377
+
378
+ # A connection may have been dropped by the server or a proxy (while
379
+ # idle, or by a restart or failover). Checking before every write
380
+ # means the command can safely go out on a fresh connection instead.
381
+ # The C extension peeks without reading; otherwise a non-blocking read
382
+ # checks, which also consumes TLS housekeeping records (like session
383
+ # tickets) that don't count. Leftover bytes mean the stream is out of
384
+ # sync.
385
+ def stale?
386
+ # @type ivar @sock: Socket | UNIXSocket | OpenSSL::SSL::SSLSocket
387
+ # @type ivar @native_io: singleton(NativeIO)
388
+ return true if @pos < @buf.bytesize
389
+ # (The native reader is only used on plain sockets.)
390
+ return @native_io.stale?(@sock) if @native_io # steep:ignore ArgumentTypeMismatch
391
+
392
+ !@sock.read_nonblock(READ_CHUNK, exception: false).instance_of?(Symbol)
393
+ rescue IOError, SystemCallError
394
+ true
395
+ end
396
+
397
+ def connect
398
+ @sock = open_socket
399
+ # Plain sockets can use the C reader; TLS ones (an OpenSSL::SSL::SSLSocket
400
+ # isn't an IO) need OpenSSL to read.
401
+ @native_io = (NATIVE_IO if @sock.is_a?(IO))
402
+ handshake
403
+ rescue SystemCallError, SocketError, IO::TimeoutError => e
404
+ close
405
+ raise ConnectionError, "#{e.class}: #{e} (#{id})"
406
+ rescue Exception # rubocop:disable Lint/RescueException
407
+ close
408
+ raise
409
+ end
410
+
411
+ def open_socket
412
+ return UNIXSocket.new(@options.fetch(:path)) if @options.fetch(:path)
413
+
414
+ host, port = @sentinels ? sentinel_master : @options.fetch_values(:host, :port)
415
+ sock = Socket.tcp(host, port, connect_timeout: @options.fetch(:connect_timeout)).tap do |s|
416
+ s.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
417
+ s.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
418
+ end
419
+ @options.fetch(:ssl) ? wrap_ssl(sock, host) : sock
420
+ end
421
+
422
+ # Asks the Sentinels, in turn, where the master is. The first one
423
+ # that answers moves to the front of the list for next time.
424
+ def sentinel_master
425
+ # @type ivar @sentinels: Array[[String, Integer]]
426
+ name = @options.fetch(:master_name)
427
+ failures = @sentinels.each_with_index.map do |(host, port), i|
428
+ address = ask_sentinel(host, port, name)
429
+ return address.tap { @sentinels.unshift(@sentinels.delete_at(i)) } if address.instance_of?(Array)
430
+
431
+ "#{host}:#{port} #{address}"
432
+ end
433
+ raise ConnectionError, "no Sentinel could locate master #{name} (#{failures.join("; ")})"
434
+ end
435
+
436
+ # The master's [host, port], or why the Sentinel couldn't tell (a
437
+ # message or an error).
438
+ def ask_sentinel(host, port, name)
439
+ sentinel = Connection.new(
440
+ host: host, port: port, username: @options.fetch(:sentinel_username),
441
+ password: @options.fetch(:sentinel_password), ssl: @options.fetch(:ssl),
442
+ ssl_params: @options.fetch(:ssl_params), connect_timeout: @options.fetch(:connect_timeout),
443
+ read_timeout: @options.fetch(:connect_timeout), write_timeout: @options.fetch(:connect_timeout)
444
+ )
445
+ begin
446
+ host, port = sentinel.call("SENTINEL", "GET-MASTER-ADDR-BY-NAME", name)
447
+ host ? [host, port] : "doesn't know it" #: [String, String] | String
448
+ rescue ConnectionError, CommandError => e
449
+ e
450
+ ensure
451
+ sentinel.close
452
+ end
453
+ end
454
+
455
+ # AUTH, SELECT and CLIENT SETNAME, pipelined. Behind Sentinel, ROLE
456
+ # confirms the server is still the master: right after a failover a
457
+ # Sentinel may briefly hand out the old one.
458
+ def handshake
459
+ commands = [] #: Array[command]
460
+ commands << ["AUTH", *@options.fetch_values(:username, :password).compact] if @options.fetch(:password)
461
+ commands << ["SELECT", @options.fetch(:db)] unless Integer(@options.fetch(:db)).zero?
462
+ commands << ["CLIENT", "SETNAME", @options.fetch(:name)] if @options.fetch(:name)
463
+ commands << ["ROLE"] if @sentinels
464
+ write(encode(commands))
465
+ replies = commands.map do
466
+ reply = read_reply(@options.fetch(:connect_timeout) + @options.fetch(:read_timeout))
467
+ raise reply if reply.instance_of?(CommandError)
468
+
469
+ reply
470
+ end
471
+ return unless @sentinels
472
+
473
+ role = replies.last.first
474
+ raise ConnectionError, "#{id} reached a #{role}, not the master" unless role == "master"
475
+ end
476
+
477
+ def wrap_ssl(sock, host)
478
+ ctx = OpenSSL::SSL::SSLContext.new
479
+ ctx.set_params(@options.fetch(:ssl_params))
480
+ # Check the hostname ourselves after the handshake, the same way on
481
+ # every engine: not every OpenSSL binding (e.g. JRuby's) honors
482
+ # verify_hostname.
483
+ check_hostname = ctx.verify_hostname && !ctx.verify_mode.equal?(OpenSSL::SSL::VERIFY_NONE)
484
+ ctx.verify_hostname = false
485
+ ssl = OpenSSL::SSL::SSLSocket.new(sock, ctx)
486
+ ssl.sync_close = true
487
+ ssl.hostname = host
488
+ loop do
489
+ state = ssl.connect_nonblock(exception: false)
490
+ break unless state.instance_of?(Symbol)
491
+
492
+ wait(sock, state, @options.fetch(:connect_timeout))
493
+ end
494
+ ssl.post_connection_check(host) if check_hostname
495
+ ssl
496
+ rescue Exception # rubocop:disable Lint/RescueException
497
+ sock.close
498
+ raise
499
+ end
500
+
501
+ # Writes +data+. If the connection breaks after part of it went out,
502
+ # the server may run some of it, so that's a ConnectionError (never
503
+ # retried); a write that sent nothing raises the original error.
504
+ def write(data)
505
+ # @type ivar @sock: Socket | UNIXSocket | OpenSSL::SSL::SSLSocket
506
+ total = data.bytesize
507
+ until data.empty?
508
+ written = @sock.write_nonblock(data, exception: false)
509
+ if written.instance_of?(Symbol)
510
+ wait(@sock, written, @options.fetch(:write_timeout))
511
+ else
512
+ data = data.byteslice(written..) #: String
513
+ end
514
+ end
515
+ rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNABORTED, IOError => e
516
+ raise if data.bytesize == total
517
+
518
+ close
519
+ raise ConnectionError, "#{e.class}: #{e} (#{id})"
520
+ end
521
+
522
+ def read_reply(timeout)
523
+ loop do
524
+ result = PARSER.parse(@buf, @pos)
525
+ return consume(result) if result
526
+
527
+ fill(timeout)
528
+ end
529
+ end
530
+
531
+ # Returns a parsed reply's value and moves past it. Once every buffered
532
+ # byte has been consumed the buffer is emptied, so a large reply's
533
+ # memory isn't held on to between commands.
534
+ def consume(result)
535
+ value, pos = result
536
+ if pos == @buf.bytesize
537
+ @buf.clear
538
+ @pos = 0
539
+ else
540
+ @pos = pos
541
+ end
542
+ value
543
+ end
544
+
545
+ # Reads whatever the server has sent, waiting up to +timeout+ for it.
546
+ def fill(timeout)
547
+ # @type ivar @sock: Socket | UNIXSocket | OpenSSL::SSL::SSLSocket
548
+ return native_fill(timeout) if @native_io
549
+
550
+ chunk = @sock.read_nonblock(READ_CHUNK, exception: false)
551
+ raise ConnectionError, "connection closed by Redis server (#{id})" unless chunk
552
+
553
+ chunk.instance_of?(Symbol) ? wait(@sock, chunk, timeout) : @buf << chunk
554
+ end
555
+
556
+ # Waits for and reads more of the reply in C (see NATIVE_IO).
557
+ def native_fill(timeout)
558
+ # @type ivar @native_io: singleton(NativeIO)
559
+ # @type ivar @sock: Socket | UNIXSocket
560
+ filled(@native_io.fill(@sock, @buf, timeout))
561
+ end
562
+
563
+ # Raises for what NativeIO.fill returns when it read nothing.
564
+ def filled(read)
565
+ raise ConnectionError, "connection closed by Redis server (#{id})" if read.nil?
566
+ raise TimeoutError, "timed out talking to Redis (#{id})" unless read
567
+ end
568
+
569
+ # Waits until +io+ is readable or writable; +kind+ is the
570
+ # :wait_readable / :wait_writable symbol a non-blocking call returned.
571
+ # A TLS socket waits on the socket under it (the openssl gem before
572
+ # 3.3, as Ruby 3.3 and earlier ship, has no wait methods of its own).
573
+ def wait(io, kind, timeout)
574
+ ready = timeout.positive? && io.to_io.public_send(kind, timeout)
575
+ raise TimeoutError, "timed out talking to Redis (#{id})" unless ready
576
+ end
577
+ end
578
+
579
+ # Collects commands for Connection#pipelined and Connection#multi.
580
+ class Pipeline
581
+ attr_reader :commands
582
+
583
+ NONE = [] #: Array[untyped]
584
+ .freeze
585
+
586
+ # +scripts+: the SHAs of the scripts the connection's server has, for
587
+ # a pipeline (see #evalsha); nil for a transaction.
588
+ def initialize(scripts = nil)
589
+ @commands = []
590
+ @scripts = scripts
591
+ # {position => SHA} of the SCRIPT LOADs it adds, made once a script
592
+ # needs loading (most pipelines don't): @loads.
593
+ end
594
+
595
+ # The positions of the SCRIPT LOADs it added, and the SHAs they load.
596
+ def loads = @loads&.keys || NONE
597
+
598
+ def loading = @loads&.values || NONE
599
+
600
+ def call(*args)
601
+ @commands << args
602
+ nil
603
+ end
604
+
605
+ # A pipeline can't recover from a NOSCRIPT mid-flight, as what it sent
606
+ # after it has run already, so a script its server may not have is
607
+ # loaded first, in the pipeline: then the script runs by its SHA
608
+ # (not resending its source every time). A server that loses its
609
+ # scripts while connected (SCRIPT FLUSH) fails the pipeline once,
610
+ # like any error, and they're loaded again the next time.
611
+ #
612
+ # A transaction sends the script itself (EVAL), always: one of its
613
+ # commands failing with NOSCRIPT would leave the rest applied.
614
+ def evalsha(script, keys = [], argv = nil)
615
+ # @type ivar @scripts: Set[String]
616
+ unless @scripts
617
+ @commands << ["EVAL", script.source, keys.size, *keys, *argv]
618
+ return
619
+ end
620
+ unless @scripts.include?(script.sha) || @loads&.value?(script.sha)
621
+ (@loads ||= {})[@commands.size] = script.sha
622
+ @commands << ["SCRIPT", "LOAD", script.source]
623
+ end
624
+ @commands << ["EVALSHA", script.sha, keys.size, *keys, *argv]
625
+ nil
626
+ end
627
+ end
628
+
629
+ # A Lua script with a precomputed SHA1 for EVALSHA.
630
+ class Script
631
+ attr_reader :source, :sha
632
+
633
+ def initialize(source)
634
+ @source = source.strip.freeze
635
+ @sha = Digest::SHA1.hexdigest(@source).freeze
636
+ end
637
+ end
638
+ end
639
+ end