robin 0.1.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +372 -0
  3. data/COPYING +674 -0
  4. data/LICENSE +165 -0
  5. data/README.md +1490 -0
  6. data/assets/favicon.svg +31 -0
  7. data/assets/robin-logo.svg +37 -0
  8. data/assets/robin-mono.svg +12 -0
  9. data/assets/robin-wordmark.svg +4 -0
  10. data/assets/robin.svg +34 -0
  11. data/exe/robin +5 -0
  12. data/ext/java/robin_resp/Blocking.java +27 -0
  13. data/ext/java/robin_resp/NativeIO.java +229 -0
  14. data/ext/java/robin_resp/NativeRESP.java +279 -0
  15. data/ext/java/robin_resp/NativeSlots.java +133 -0
  16. data/ext/java/robin_resp/RobinRespService.java +32 -0
  17. data/lib/active_job/queue_adapters/robin_adapter.rb +3 -0
  18. data/lib/generators/robin/install/install_generator.rb +39 -0
  19. data/lib/generators/robin/install/templates/initializer.rb.tt +31 -0
  20. data/lib/generators/robin/install/templates/robin.yml.tt +25 -0
  21. data/lib/generators/robin/job/job_generator.rb +78 -0
  22. data/lib/generators/robin/job/templates/job.rb.tt +13 -0
  23. data/lib/generators/robin/job/templates/job_spec.rb.tt +21 -0
  24. data/lib/generators/robin/job/templates/job_test.rb.tt +21 -0
  25. data/lib/puma/plugin/robin.rb +131 -0
  26. data/lib/robin/active_job.rb +144 -0
  27. data/lib/robin/api.rb +777 -0
  28. data/lib/robin/batch.rb +700 -0
  29. data/lib/robin/cancellation.rb +62 -0
  30. data/lib/robin/cli.rb +325 -0
  31. data/lib/robin/client.rb +372 -0
  32. data/lib/robin/config.rb +461 -0
  33. data/lib/robin/cron.rb +325 -0
  34. data/lib/robin/debounce.rb +108 -0
  35. data/lib/robin/deploy.rb +65 -0
  36. data/lib/robin/embedded.rb +53 -0
  37. data/lib/robin/encryption.rb +147 -0
  38. data/lib/robin/events.rb +60 -0
  39. data/lib/robin/expiry.rb +41 -0
  40. data/lib/robin/fetcher.rb +441 -0
  41. data/lib/robin/gvl.rb +62 -0
  42. data/lib/robin/health.rb +76 -0
  43. data/lib/robin/iterable_job.rb +294 -0
  44. data/lib/robin/job.rb +258 -0
  45. data/lib/robin/job_class.rb +157 -0
  46. data/lib/robin/job_logger.rb +121 -0
  47. data/lib/robin/launcher.rb +658 -0
  48. data/lib/robin/leader.rb +86 -0
  49. data/lib/robin/limiter.rb +639 -0
  50. data/lib/robin/manager.rb +125 -0
  51. data/lib/robin/metrics.rb +303 -0
  52. data/lib/robin/middleware.rb +165 -0
  53. data/lib/robin/opentelemetry.rb +88 -0
  54. data/lib/robin/periodic.rb +327 -0
  55. data/lib/robin/processor.rb +317 -0
  56. data/lib/robin/profiler.rb +140 -0
  57. data/lib/robin/progress.rb +46 -0
  58. data/lib/robin/rails/app_loader.rb +23 -0
  59. data/lib/robin/rails/current_attributes.rb +88 -0
  60. data/lib/robin/rails/error_reporter.rb +31 -0
  61. data/lib/robin/rails/notifications.rb +22 -0
  62. data/lib/robin/rails/railtie.rb +87 -0
  63. data/lib/robin/rails/transactional_push.rb +49 -0
  64. data/lib/robin/rails.rb +23 -0
  65. data/lib/robin/recovery.rb +135 -0
  66. data/lib/robin/redis/connection.rb +639 -0
  67. data/lib/robin/redis/pool.rb +190 -0
  68. data/lib/robin/redis/resp.rb +77 -0
  69. data/lib/robin/reliable_push.rb +153 -0
  70. data/lib/robin/retrier.rb +284 -0
  71. data/lib/robin/scheduler.rb +102 -0
  72. data/lib/robin/scripts.rb +331 -0
  73. data/lib/robin/statsd.rb +135 -0
  74. data/lib/robin/swarm.rb +319 -0
  75. data/lib/robin/systemd.rb +47 -0
  76. data/lib/robin/testing.rb +158 -0
  77. data/lib/robin/top.rb +178 -0
  78. data/lib/robin/unique.rb +198 -0
  79. data/lib/robin/version.rb +5 -0
  80. data/lib/robin/warmup.rb +94 -0
  81. data/lib/robin/web/api.rb +580 -0
  82. data/lib/robin/web/batch_status.rb +44 -0
  83. data/lib/robin/web/dist/assets/ar-y7v9b75f.js +1 -0
  84. data/lib/robin/web/dist/assets/ar-y7v9b75f.js.gz +0 -0
  85. data/lib/robin/web/dist/assets/bricolage-0qmzcpnj.woff2 +0 -0
  86. data/lib/robin/web/dist/assets/cs-df88mq5e.js +1 -0
  87. data/lib/robin/web/dist/assets/cs-df88mq5e.js.gz +0 -0
  88. data/lib/robin/web/dist/assets/da-kemms74g.js +1 -0
  89. data/lib/robin/web/dist/assets/da-kemms74g.js.gz +0 -0
  90. data/lib/robin/web/dist/assets/de-z8kkqg65.js +1 -0
  91. data/lib/robin/web/dist/assets/de-z8kkqg65.js.gz +0 -0
  92. data/lib/robin/web/dist/assets/es-9ahrrn1s.js +1 -0
  93. data/lib/robin/web/dist/assets/es-9ahrrn1s.js.gz +0 -0
  94. data/lib/robin/web/dist/assets/fa-7r0x28nc.js +1 -0
  95. data/lib/robin/web/dist/assets/fa-7r0x28nc.js.gz +0 -0
  96. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg +31 -0
  97. data/lib/robin/web/dist/assets/favicon-1ebfcc0182.svg.gz +0 -0
  98. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js +1 -0
  99. data/lib/robin/web/dist/assets/fr-r7mqf3d9.js.gz +0 -0
  100. data/lib/robin/web/dist/assets/he-7t46b6rt.js +1 -0
  101. data/lib/robin/web/dist/assets/he-7t46b6rt.js.gz +0 -0
  102. data/lib/robin/web/dist/assets/hi-zj434tx4.js +1 -0
  103. data/lib/robin/web/dist/assets/hi-zj434tx4.js.gz +0 -0
  104. data/lib/robin/web/dist/assets/it-pqc2m7gm.js +1 -0
  105. data/lib/robin/web/dist/assets/it-pqc2m7gm.js.gz +0 -0
  106. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js +1 -0
  107. data/lib/robin/web/dist/assets/ja-k9rgm1pj.js.gz +0 -0
  108. data/lib/robin/web/dist/assets/ko-bxersv91.js +1 -0
  109. data/lib/robin/web/dist/assets/ko-bxersv91.js.gz +0 -0
  110. data/lib/robin/web/dist/assets/main-anw3f6ba.css +1 -0
  111. data/lib/robin/web/dist/assets/main-anw3f6ba.css.gz +0 -0
  112. data/lib/robin/web/dist/assets/main-pdtvjejb.js +7 -0
  113. data/lib/robin/web/dist/assets/main-pdtvjejb.js.gz +0 -0
  114. data/lib/robin/web/dist/assets/main-zz8zzhwn.js +2 -0
  115. data/lib/robin/web/dist/assets/main-zz8zzhwn.js.gz +0 -0
  116. data/lib/robin/web/dist/assets/nb-fjnc5064.js +1 -0
  117. data/lib/robin/web/dist/assets/nb-fjnc5064.js.gz +0 -0
  118. data/lib/robin/web/dist/assets/nl-9p8b95b6.js +1 -0
  119. data/lib/robin/web/dist/assets/nl-9p8b95b6.js.gz +0 -0
  120. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js +1 -0
  121. data/lib/robin/web/dist/assets/pl-8xsrdn8n.js.gz +0 -0
  122. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js +1 -0
  123. data/lib/robin/web/dist/assets/pt-BR-650j7cvg.js.gz +0 -0
  124. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg +34 -0
  125. data/lib/robin/web/dist/assets/robin-ae4a4aedd9.svg.gz +0 -0
  126. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg +4 -0
  127. data/lib/robin/web/dist/assets/robin-wordmark-5dde5d8e4b.svg.gz +0 -0
  128. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js +1 -0
  129. data/lib/robin/web/dist/assets/ru-a9ftyw3d.js.gz +0 -0
  130. data/lib/robin/web/dist/assets/sv-j9gqamr1.js +1 -0
  131. data/lib/robin/web/dist/assets/sv-j9gqamr1.js.gz +0 -0
  132. data/lib/robin/web/dist/assets/tr-y9z0q51g.js +1 -0
  133. data/lib/robin/web/dist/assets/tr-y9z0q51g.js.gz +0 -0
  134. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js +1 -0
  135. data/lib/robin/web/dist/assets/uk-2dr9ew3p.js.gz +0 -0
  136. data/lib/robin/web/dist/assets/ur-h25nzq1s.js +1 -0
  137. data/lib/robin/web/dist/assets/ur-h25nzq1s.js.gz +0 -0
  138. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js +1 -0
  139. data/lib/robin/web/dist/assets/vi-9y2ff1bd.js.gz +0 -0
  140. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js +1 -0
  141. data/lib/robin/web/dist/assets/zh-CN-tbj9mf98.js.gz +0 -0
  142. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js +1 -0
  143. data/lib/robin/web/dist/assets/zh-TW-59q8j9vw.js.gz +0 -0
  144. data/lib/robin/web/dist/index.html +19 -0
  145. data/lib/robin/web/extension.rb +193 -0
  146. data/lib/robin/web/permissions.rb +40 -0
  147. data/lib/robin/web/prometheus.rb +79 -0
  148. data/lib/robin/web/router.rb +150 -0
  149. data/lib/robin/web/static.rb +110 -0
  150. data/lib/robin/web.rb +196 -0
  151. data/lib/robin.rb +261 -0
  152. data/lib/robin_resp/robin_resp.jar +0 -0
  153. data/sig/manifest.yaml +9 -0
  154. data/sig/robin/api.rbs +442 -0
  155. data/sig/robin/batch.rbs +292 -0
  156. data/sig/robin/cancellation.rbs +25 -0
  157. data/sig/robin/cli.rbs +102 -0
  158. data/sig/robin/client.rbs +118 -0
  159. data/sig/robin/config.rbs +341 -0
  160. data/sig/robin/cron.rbs +149 -0
  161. data/sig/robin/debounce.rbs +36 -0
  162. data/sig/robin/deploy.rbs +24 -0
  163. data/sig/robin/embedded.rbs +25 -0
  164. data/sig/robin/encryption.rbs +49 -0
  165. data/sig/robin/events.rbs +10 -0
  166. data/sig/robin/expiry.rbs +17 -0
  167. data/sig/robin/fetcher.rbs +134 -0
  168. data/sig/robin/gvl.rbs +39 -0
  169. data/sig/robin/health.rbs +30 -0
  170. data/sig/robin/iterable_job.rbs +99 -0
  171. data/sig/robin/job.rbs +173 -0
  172. data/sig/robin/job_class.rbs +55 -0
  173. data/sig/robin/job_logger.rbs +56 -0
  174. data/sig/robin/launcher.rbs +191 -0
  175. data/sig/robin/leader.rbs +43 -0
  176. data/sig/robin/limiter.rbs +317 -0
  177. data/sig/robin/manager.rbs +54 -0
  178. data/sig/robin/metrics.rbs +122 -0
  179. data/sig/robin/middleware.rbs +106 -0
  180. data/sig/robin/opentelemetry.rbs +51 -0
  181. data/sig/robin/periodic.rbs +81 -0
  182. data/sig/robin/processor.rbs +107 -0
  183. data/sig/robin/profiler.rbs +60 -0
  184. data/sig/robin/progress.rbs +22 -0
  185. data/sig/robin/rails/app_loader.rbs +10 -0
  186. data/sig/robin/rails/current_attributes.rbs +44 -0
  187. data/sig/robin/rails/error_reporter.rbs +21 -0
  188. data/sig/robin/rails/notifications.rbs +9 -0
  189. data/sig/robin/rails/transactional_push.rbs +16 -0
  190. data/sig/robin/rails.rbs +8 -0
  191. data/sig/robin/recovery.rbs +29 -0
  192. data/sig/robin/redis/connection.rbs +224 -0
  193. data/sig/robin/redis/pool.rbs +89 -0
  194. data/sig/robin/redis/resp.rbs +29 -0
  195. data/sig/robin/reliable_push.rbs +77 -0
  196. data/sig/robin/retrier.rbs +72 -0
  197. data/sig/robin/scheduler.rbs +40 -0
  198. data/sig/robin/scripts.rbs +36 -0
  199. data/sig/robin/statsd.rbs +55 -0
  200. data/sig/robin/swarm.rbs +133 -0
  201. data/sig/robin/systemd.rbs +13 -0
  202. data/sig/robin/testing.rbs +80 -0
  203. data/sig/robin/top.rbs +79 -0
  204. data/sig/robin/unique.rbs +72 -0
  205. data/sig/robin/version.rbs +3 -0
  206. data/sig/robin/warmup.rbs +44 -0
  207. data/sig/robin/web/api.rbs +196 -0
  208. data/sig/robin/web/batch_status.rbs +25 -0
  209. data/sig/robin/web/extension.rbs +106 -0
  210. data/sig/robin/web/permissions.rbs +14 -0
  211. data/sig/robin/web/prometheus.rbs +33 -0
  212. data/sig/robin/web/router.rbs +53 -0
  213. data/sig/robin/web/static.rbs +42 -0
  214. data/sig/robin/web.rbs +140 -0
  215. data/sig/robin.rbs +116 -0
  216. data/sig/robin_resp.rbs +55 -0
  217. metadata +294 -0
@@ -0,0 +1,190 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ module Redis
5
+ # A small, thread-safe connection pool. Nested #with calls from the same
6
+ # fiber reuse the connection that's already checked out.
7
+ class Pool
8
+ class TimeoutError < Robin::Redis::TimeoutError; end
9
+
10
+ # The pool's bookkeeping: how many more connections may be checked out,
11
+ # and the idle ones, most recently returned last. The extension's
12
+ # NativeSlots does the same without a lock (JRuby, where threads run
13
+ # in parallel and a shared lock would make every checkout wait).
14
+ class Slots
15
+ def initialize(size)
16
+ @mutex = Mutex.new
17
+ @cond = ConditionVariable.new
18
+ @free = size
19
+ @idle = [] # [connection, idle since], least recently used first
20
+ end
21
+
22
+ # An idle connection, :create (a slot to fill with a new one), or
23
+ # nil after waiting +timeout+ seconds for either.
24
+ def take(timeout)
25
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
26
+ @mutex.synchronize do
27
+ until @free.positive?
28
+ remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
29
+ return unless remaining.positive?
30
+
31
+ @cond.wait(@mutex, remaining)
32
+ end
33
+ @free -= 1
34
+ @idle.pop&.first || :create #: Connection | :create
35
+ end
36
+ end
37
+
38
+ # Returns a connection, idle since +since+ (monotonic seconds).
39
+ def put(conn, since)
40
+ @mutex.synchronize do
41
+ @idle.push([conn, since])
42
+ @free += 1
43
+ @cond.signal
44
+ end
45
+ end
46
+
47
+ # Frees a slot taken with :create but never filled.
48
+ def release
49
+ @mutex.synchronize do
50
+ @free += 1
51
+ @cond.signal
52
+ end
53
+ end
54
+
55
+ # Makes +count+ more connections available.
56
+ def add(count)
57
+ @mutex.synchronize do
58
+ @free += count
59
+ @cond.broadcast
60
+ end
61
+ end
62
+
63
+ # Removes and returns the connections idle since before +cutoff+.
64
+ def reap(cutoff)
65
+ @mutex.synchronize do
66
+ stale = @idle.take_while { |_conn, since| since < cutoff }
67
+ @idle.shift(stale.size).map(&:first)
68
+ end
69
+ end
70
+
71
+ # Removes and returns every idle connection.
72
+ def drain
73
+ @mutex.synchronize do
74
+ conns = @idle.map(&:first)
75
+ @idle.clear
76
+ conns
77
+ end
78
+ end
79
+ end
80
+
81
+ # The extension's slots where there are some (JRuby), else Slots.
82
+ def self.slots_class(redis = Redis) = redis.const_defined?(:NativeSlots) ? redis::NativeSlots : Slots
83
+
84
+ SLOTS = slots_class
85
+
86
+ attr_reader :size, :timeout, :idle_timeout
87
+
88
+ # +idle_timeout+ (seconds) closes connections nobody has used for that
89
+ # long, the next time one is checked out, so a mostly idle process
90
+ # doesn't hold on to Redis connections. (Redis's own `timeout`
91
+ # setting does the same from the server side; Robin reconnects
92
+ # transparently either way.)
93
+ def initialize(size: 5, timeout: 3.0, idle_timeout: nil, &factory)
94
+ raise ArgumentError, "a connection factory block is required" unless factory
95
+
96
+ @size = size
97
+ @timeout = timeout
98
+ @idle_timeout = idle_timeout
99
+ @factory = factory
100
+ @slots = SLOTS.new(size)
101
+ @key = :"robin_pool_#{object_id}"
102
+ @pid = Process.pid
103
+ @reset = Mutex.new
104
+ end
105
+
106
+ def with
107
+ if (conn = Thread.current[@key])
108
+ return yield(conn)
109
+ end
110
+
111
+ conn = checkout
112
+ Thread.current[@key] = conn
113
+ begin
114
+ yield conn
115
+ ensure
116
+ Thread.current[@key] = nil
117
+ checkin(conn)
118
+ end
119
+ end
120
+
121
+ # Convenience pass-through so a pool can be used like a connection for
122
+ # single commands.
123
+ def call(*args) = with { |c| c.call(*args) }
124
+
125
+ def pipelined(&) = with { |c| c.pipelined(&) }
126
+
127
+ def multi(&) = with { |c| c.multi(&) }
128
+
129
+ def evalsha(script, keys = [], argv = []) = with { |c| c.evalsha(script, keys, argv) }
130
+
131
+ def blocking_call(timeout, *args) = with { |c| c.blocking_call(timeout, *args) }
132
+
133
+ def shutdown
134
+ reset_after_fork
135
+ @slots.drain.each(&:close)
136
+ end
137
+
138
+ # Grows the pool to +size+ connections; a smaller size changes
139
+ # nothing. Returns the pool's size.
140
+ def grow(size)
141
+ return @size if size <= @size
142
+
143
+ @slots.add(size - @size)
144
+ @size = size
145
+ end
146
+
147
+ private
148
+
149
+ def checkout
150
+ # @type ivar @idle_timeout: seconds
151
+ reset_after_fork
152
+ @slots.reap(monotonic - @idle_timeout).each(&:close) if @idle_timeout
153
+ conn = @slots.take(@timeout)
154
+ raise TimeoutError, "waited #{@timeout}s for a Redis connection (pool size #{@size})" unless conn
155
+ # Or :create, a slot to fill with a new connection.
156
+ return conn unless conn.instance_of?(Symbol)
157
+
158
+ begin
159
+ @factory.call
160
+ rescue Exception # rubocop:disable Lint/RescueException
161
+ @slots.release
162
+ raise
163
+ end
164
+ end
165
+
166
+ # Idle times only matter with an idle timeout; 0 stands in otherwise.
167
+ def checkin(conn) = @slots.put(conn, @idle_timeout ? monotonic : 0)
168
+
169
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
170
+
171
+ # Sockets must never be shared between a parent and a forked child,
172
+ # and locks held by the parent's other threads would never be freed.
173
+ # The child abandons what it inherited (see Connection#abandon). Its
174
+ # threads may all notice the fork at once: one resets the pool, and
175
+ # the rest find it done (two resets would lose the first one's
176
+ # checkouts, and hand out more connections than the pool holds).
177
+ def reset_after_fork
178
+ return if @pid == Process.pid
179
+
180
+ @reset.synchronize do
181
+ return if @pid == Process.pid
182
+
183
+ @slots.drain.each(&:abandon)
184
+ @slots = SLOTS.new(@size)
185
+ @pid = Process.pid
186
+ end
187
+ end
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ module Redis
5
+ # Raised for "-ERR ..." replies. Error replies nested inside arrays (for
6
+ # example, inside an EXEC result) are returned as instances rather than
7
+ # raised so callers can decide what to do with them.
8
+ class CommandError < StandardError; end
9
+
10
+ class ProtocolError < StandardError; end
11
+
12
+ # Encodes commands and decodes replies in the Redis serialization
13
+ # protocol, version 2 (Robin never switches a connection to RESP3).
14
+ #
15
+ # The decoder works on an in-memory buffer: +parse(buffer, offset)+
16
+ # returns +[value, next_offset]+ or +nil+ when the buffer does not yet
17
+ # hold a complete reply. That contract lets the optional C extension
18
+ # (Robin::Redis::NativeRESP) be swapped in transparently.
19
+ module RESP
20
+ CRLF = "\r\n"
21
+
22
+ # Appends the command +args+ (anything responding to #to_s) to +out+.
23
+ def self.encode(args, out = "".b)
24
+ out << "*" << args.size.to_s << CRLF
25
+ args.each do |arg|
26
+ str = arg.to_s
27
+ out << "$" << str.bytesize.to_s << CRLF << str.b << CRLF
28
+ end
29
+ out
30
+ end
31
+
32
+ def self.parse(buf, pos)
33
+ eol = buf.byteindex(CRLF, pos)
34
+ return unless eol
35
+
36
+ line = buf.byteslice(pos + 1, eol - pos - 1) #: String
37
+ nxt = eol + 2
38
+ case buf.getbyte(pos)
39
+ when 43 then [utf8(line), nxt] # +
40
+ when 45 then [CommandError.new(utf8(line)), nxt] # -
41
+ when 58 then [Integer(line), nxt] # :
42
+ when 36 then parse_bulk(buf, nxt, Integer(line)) # $
43
+ when 42 then parse_array(buf, nxt, Integer(line)) # *
44
+ else raise ProtocolError, "unknown RESP type byte #{buf.getbyte(pos)}"
45
+ end
46
+ end
47
+
48
+ def self.parse_bulk(buf, pos, len)
49
+ return [nil, pos] if len.negative?
50
+
51
+ stop = pos + len
52
+ return if buf.bytesize < stop + 2
53
+
54
+ # (The buffer holds len more bytes, so the slice isn't nil.)
55
+ [utf8(buf.byteslice(pos, len)), stop + 2] # steep:ignore ArgumentTypeMismatch
56
+ end
57
+
58
+ def self.parse_array(buf, pos, count)
59
+ return [nil, pos] if count.negative?
60
+
61
+ items = [] #: Array[reply]
62
+ count.times do
63
+ result = parse(buf, pos)
64
+ return unless result
65
+
66
+ value, pos = result
67
+ items << value
68
+ end
69
+ [items, pos]
70
+ end
71
+
72
+ def self.utf8(str) = str.force_encoding(Encoding::UTF_8)
73
+
74
+ private_class_method :parse_bulk, :parse_array, :utf8
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Robin
4
+ # Keeps pushes from failing while Redis is briefly unreachable.
5
+ #
6
+ # Robin.configure { |c| c.reliable_push = true } # or a buffer size
7
+ #
8
+ # When a push fails with a connection error, its jobs are buffered in
9
+ # memory and pushed (in order) once Redis is back: by a background
10
+ # thread retrying with backoff, or by the next successful push. The
11
+ # buffer is bounded (1000 jobs by default); when it's full the oldest
12
+ # jobs are dropped. When the process exits, it tries once more to push
13
+ # what's buffered; jobs it can't push are lost. Dropped and lost jobs are
14
+ # reported to config.error_handlers (as Dropped and Lost errors) and as
15
+ # push.dropped and push.lost events.
16
+ class ReliablePush
17
+ DEFAULT_LIMIT = 1_000
18
+ DRAIN_CHUNK = 100 # jobs per pipelined push while draining
19
+ ERRORS = [Redis::ConnectionError, Redis::Pool::TimeoutError].freeze
20
+
21
+ # Jobs dropped because the buffer was full.
22
+ class Dropped < Error; end
23
+
24
+ # Jobs still buffered when the process exited.
25
+ class Lost < Error; end
26
+
27
+ @instances = {}
28
+ @lock = Mutex.new
29
+ AT_EXIT = ->(reliable_push) { Kernel.at_exit { reliable_push.finish } }
30
+ @exit_hook = AT_EXIT
31
+
32
+ class << self
33
+ # Called with each instance the first time it buffers, to have it
34
+ # #finish when the process exits.
35
+ attr_accessor :exit_hook
36
+ end
37
+
38
+ # One buffer per config and Redis pool (see Client.via), so buffered
39
+ # jobs go back to the Redis they were meant for.
40
+ def self.for(config, pool = nil)
41
+ @lock.synchronize { @instances[[config.object_id, pool.object_id]] ||= new(config, pool: pool) }
42
+ end
43
+
44
+ attr_reader :limit
45
+
46
+ def initialize(config, limit: nil, pool: nil)
47
+ @config = config
48
+ @pool = pool
49
+ @limit = limit || (config.reliable_push.instance_of?(Integer) ? config.reliable_push : DEFAULT_LIMIT)
50
+ @buffer = []
51
+ @mutex = Mutex.new
52
+ end
53
+
54
+ def size = @mutex.synchronize { @buffer.size }
55
+
56
+ # Yields to push +payloads+; buffers them if Redis is unreachable.
57
+ def push(payloads)
58
+ if @buffer.empty?
59
+ begin
60
+ return yield
61
+ rescue *ERRORS => e
62
+ @config.logger.warn("Redis unavailable (#{e}); buffering #{payloads.size} job(s) for later")
63
+ Events.emit(@config, "push.buffered", jobs: payloads, error: e)
64
+ end
65
+ end
66
+ dropped = @mutex.synchronize { append(payloads) }
67
+ # Reported outside the lock: an error handler may push a job itself.
68
+ report_dropped(dropped) unless dropped.empty?
69
+ drain
70
+ nil
71
+ end
72
+
73
+ # Pushes buffered jobs, oldest first. Returns true if the buffer is empty.
74
+ def drain
75
+ pushed = 0
76
+ done = @mutex.synchronize do
77
+ loop do
78
+ break true if @buffer.empty?
79
+
80
+ chunk = @buffer.first(DRAIN_CHUNK)
81
+ begin
82
+ push_chunk(chunk)
83
+ rescue *ERRORS
84
+ start_retrier
85
+ break false
86
+ end
87
+ @buffer.shift(chunk.size)
88
+ pushed += chunk.size
89
+ end
90
+ end
91
+ report_pushed(pushed) if pushed > 0
92
+ done
93
+ end
94
+
95
+ # Run when the process exits: one last try to push what's buffered.
96
+ # Whatever is still buffered after it is lost, and reported.
97
+ def finish
98
+ return if drain
99
+
100
+ lost = @mutex.synchronize { @buffer.tap { @buffer = [] } }
101
+ @config.handle_exception(Lost.new("process exiting with #{lost.size} buffered job(s) not pushed: #{describe(lost)}"),
102
+ {context: "reliable push"})
103
+ Events.emit(@config, "push.lost", jobs: lost)
104
+ end
105
+
106
+ private
107
+
108
+ # Returns the jobs it dropped to stay within the limit.
109
+ def append(payloads)
110
+ unless @hooked
111
+ @hooked = true
112
+ ReliablePush.exit_hook.call(self)
113
+ end
114
+ @buffer.concat(payloads)
115
+ @buffer.shift([@buffer.size - @limit, 0].max)
116
+ end
117
+
118
+ def report_dropped(dropped)
119
+ @config.handle_exception(Dropped.new("reliable push buffer full (#{@limit}); dropped #{dropped.size} job(s): #{describe(dropped)}"),
120
+ {context: "reliable push"})
121
+ Events.emit(@config, "push.dropped", jobs: dropped)
122
+ end
123
+
124
+ def report_pushed(count)
125
+ @config.logger.info("Redis is reachable again; pushed #{count} buffered job(s)")
126
+ @config.statsd&.increment("jobs.recovered.push", count)
127
+ Events.emit(@config, "push.recovered", count: count)
128
+ end
129
+
130
+ def push_chunk(chunk) = Client.new(config: @config, pool: @pool).__send__(:raw_push, chunk)
131
+
132
+ def describe(jobs) = jobs.map { |p| "#{p.fetch("class")} jid=#{p.fetch("jid")}" }.join(", ")
133
+
134
+ # Must be called with @mutex held.
135
+ def start_retrier
136
+ return if @retrier&.alive?
137
+
138
+ @retrier = Thread.new do
139
+ Thread.current.name = "robin.reliable_push"
140
+ delay = 0.5 #: seconds
141
+ loop do
142
+ pause(delay)
143
+ break if drain
144
+
145
+ delay = [delay * 2, 30].min
146
+ end
147
+ @retrier = nil
148
+ end
149
+ end
150
+
151
+ def pause(seconds) = sleep(seconds)
152
+ end
153
+ end
@@ -0,0 +1,284 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "zlib"
4
+
5
+ module Robin
6
+ # Decides what happens to a job that raised: schedule a retry with
7
+ # exponential backoff, move it to the dead set, or discard it.
8
+ class Retrier
9
+ MAX_MESSAGE_BYTES = 10_000
10
+ MAX_DELAY = 7 * 86_400
11
+ # Earlier versions marked compressed backtraces with this.
12
+ BACKTRACE_PREFIX = "deflate:"
13
+
14
+ # Backtraces are stored compressed, as the base64 of the deflated JSON
15
+ # lines: they're often most of a failed job's size, and the retry and
16
+ # dead sets can hold a lot of failed jobs.
17
+ def self.pack_backtrace(lines)
18
+ return lines if lines.empty?
19
+
20
+ [Zlib.deflate(Robin.dump_json(lines))].pack("m0")
21
+ end
22
+
23
+ # A stored error_backtrace (compressed or a plain array) as an array
24
+ # of lines.
25
+ def self.unpack_backtrace(value)
26
+ return value unless value.instance_of?(String)
27
+
28
+ # (unpack1("m0") returns a String; Steep can't tell from the format.)
29
+ deflated = value.delete_prefix(BACKTRACE_PREFIX).unpack1("m0") #: String
30
+ Robin.load_json(Zlib.inflate(deflated))
31
+ rescue Zlib::Error, ArgumentError, JSON::ParserError
32
+ ["(unreadable backtrace)"]
33
+ end
34
+
35
+ attr_reader :config
36
+
37
+ def initialize(config)
38
+ @config = config
39
+ end
40
+
41
+ # Default backoff: ~15s, 26s, 45s, ... growing by 1.7x per failure,
42
+ # capped at a week, with +/-20% jitter so retries of a large failed
43
+ # batch don't stampede. 20 retries span roughly 10 days.
44
+ def self.default_delay(failures)
45
+ base = [15 * (1.7**(failures - 1)), MAX_DELAY].min
46
+ (base * (0.8 + rand * 0.4)).round
47
+ end
48
+
49
+ # A custom delay pushed up to 10% (at most 10s) later, so jobs that
50
+ # failed together don't all retry in the same instant.
51
+ def self.jitter(seconds)
52
+ seconds + rand * [seconds * 0.1, 10].min
53
+ end
54
+
55
+ # An exception's message as valid UTF-8, even when reading it raises.
56
+ def self.message_of(exception)
57
+ Robin.utf8(exception.message.to_s)
58
+ rescue => e
59
+ "(#{e.class} raised reading the error message)"
60
+ end
61
+
62
+ # A job with +retry_for+ and no explicit retry count retries until its
63
+ # time is up, however many attempts that takes.
64
+ def max_retries(payload)
65
+ case payload["retry"]
66
+ when true, nil then payload["retry_for"] ? Float::INFINITY : config.max_retries
67
+ when false then 0
68
+ else Integer(payload.fetch("retry"))
69
+ end
70
+ end
71
+
72
+ # Fields a failure adds to a job before it's written back to Redis.
73
+ FAILURE_FIELDS = %w[error_class error_message error_backtrace failed_at retried_at retry_count discarded_at].freeze
74
+
75
+ # Records the failure and acknowledges the unit of work. Returns the
76
+ # outcome: :retry, :dead or :discard. Whatever the exception or the
77
+ # job's hooks do, only Redis errors make this raise (and the job then
78
+ # stays in flight).
79
+ #
80
+ # The failure details are added to +payload+ (for hooks and handlers),
81
+ # but what's written back to Redis is the job as it was stored plus
82
+ # those details, so arguments the job or middleware changed in memory
83
+ # (or decrypted) never are.
84
+ #
85
+ # +batch_failure+ is the failure the job's batch middleware held back
86
+ # (see Batch.holding_failure): it's recorded with the retry, in the
87
+ # same transaction. A job that won't run again records its failure
88
+ # with the batch anyway.
89
+ def handle(unit, payload, job_class, exception, batch_failure: nil)
90
+ payload["error_class"] = exception.class.name
91
+ payload["error_message"] = truncate(self.class.message_of(exception))
92
+ now = Robin.now_ms
93
+ payload["failed_at"] ||= now
94
+ # retry_count is 0 after the first failure, then counts the retries
95
+ # that failed too.
96
+ if payload["retry_count"]
97
+ payload["retried_at"] = now
98
+ payload["retry_count"] = Integer(payload.fetch("retry_count")) + 1
99
+ else
100
+ payload["retry_count"] = 0
101
+ end
102
+ if (lines = backtrace_lines(payload))
103
+ payload["error_backtrace"] = self.class.pack_backtrace(backtrace_of(exception, lines))
104
+ end
105
+ return discard(unit, payload, exception) if payload["retry"] == false
106
+
107
+ failures = payload.fetch("retry_count") + 1
108
+ delay = (failures <= max_retries(payload)) ? retry_delay(job_class, failures, exception, payload) : :kill
109
+ at = now + Integer(delay * 1000) if delay.is_a?(Numeric)
110
+ delay = :kill if at && out_of_time?(payload, at)
111
+
112
+ case delay
113
+ when :discard
114
+ discard(unit, payload, exception)
115
+ when :kill
116
+ die(unit, payload, job_class, exception)
117
+ else
118
+ # (at is set when delay is a number of seconds, as it is here.)
119
+ payload["at"] = Robin.epoch_seconds(at) # steep:ignore ArgumentTypeMismatch
120
+ entry = stored(payload)
121
+ entry["queue"] = payload["queue"] = payload.fetch("retry_queue") if payload["retry_queue"]
122
+ replies = config.redis do |conn|
123
+ conn.multi do |tx|
124
+ Batch.queue_failure(tx, config, payload, batch_failure) if batch_failure
125
+ tx.call("ZADD", config.key("retry"), payload.fetch("at"), Robin.dump_json(entry))
126
+ unit.acknowledge(tx)
127
+ end
128
+ end
129
+ Batch.updated(config, payload.fetch("bid"), replies.first) if batch_failure
130
+ :retry
131
+ end
132
+ end
133
+
134
+ # Moves a job to the dead set, trimming it by age and size, then runs
135
+ # the class's retries_exhausted hook and the global death handlers. The
136
+ # hook may return :discard to drop the job instead (it isn't kept in
137
+ # the dead set; handlers still run, with "discarded_at" set).
138
+ def die(unit, payload, job_class, exception)
139
+ if discard_requested?(job_class, payload, exception) || payload["dead"] == false
140
+ discard(unit, payload, exception)
141
+ else
142
+ raw = Robin.dump_json(stored(payload))
143
+ terminal(payload, exception) { |tx| DeadSet.new(config: config).queue_kill(tx, raw, unit) }
144
+ :dead
145
+ end
146
+ end
147
+
148
+ private
149
+
150
+ # Drops a job for good: acknowledges it, records when, and tells its
151
+ # batch and the death handlers that it won't run again.
152
+ def discard(unit, payload, exception)
153
+ payload["discarded_at"] = Robin.now_ms
154
+ terminal(payload, exception) { |tx| unit&.acknowledge(tx) }
155
+ :discard
156
+ end
157
+
158
+ # A job that won't run again: in one transaction, it goes where the
159
+ # block (yielded the transaction) puts it and is acknowledged, gives
160
+ # up its unique lock (if it has one), and tells its batch (if any);
161
+ # then the batch callbacks this calls for and the death handlers run.
162
+ def terminal(payload, exception)
163
+ bid = payload["bid"]
164
+ replies = config.redis do |conn|
165
+ conn.multi do |tx|
166
+ yield tx
167
+ Unique.eval_release(tx, config, payload)
168
+ Batch.queue_death(tx, config, payload, exception) if bid
169
+ end
170
+ end
171
+ Batch.death_recorded(config, bid, replies.last(2)) if bid
172
+ config.death_handlers.each do |handler|
173
+ handler.call(payload, exception)
174
+ rescue => e
175
+ config.handle_exception(e, {context: "death handler", job: payload})
176
+ end
177
+ end
178
+
179
+ # Runs the class's retries_exhausted hook; true when it asks for the
180
+ # job to be discarded instead of kept in the dead set.
181
+ def discard_requested?(job_class, payload, exception)
182
+ hooks = hook_class(job_class, payload)
183
+ return false unless hooks.respond_to?(:robin_retries_exhausted_block)
184
+
185
+ hooks.robin_retries_exhausted_block&.call(payload, exception) == :discard
186
+ rescue => e
187
+ config.handle_exception(e, {context: "retries_exhausted hook", job: payload})
188
+ false
189
+ end
190
+
191
+ # What's written to Redis: the job as stored, plus the failure details.
192
+ def stored(payload)
193
+ Robin.pristine(payload).except("at", "enqueued_at").merge(payload.slice(*FAILURE_FIELDS))
194
+ end
195
+
196
+ # Would a retry at +at+ (ms) come more than +retry_for+ seconds after
197
+ # the job first failed?
198
+ def out_of_time?(payload, at)
199
+ window = payload["retry_for"]
200
+ window && at > Robin.ms_of(payload.fetch("failed_at")) + window * 1000
201
+ end
202
+
203
+ # Seconds until the retry, :discard or :kill: the class's
204
+ # robin_retry_in answer (jittered), else the default backoff; or what
205
+ # config.retry_backoff makes of either.
206
+ def retry_delay(job_class, failures, exception, payload)
207
+ hooks = hook_class(job_class, payload)
208
+ block = hooks.respond_to?(:robin_retry_in_block) && hooks.robin_retry_in_block
209
+ custom = custom_delay(block, failures, exception, payload) if block
210
+ return custom if custom.instance_of?(Symbol)
211
+ return config.retry_backoff.call(failures, exception, payload, custom) if config.retry_backoff
212
+ return self.class.default_delay(failures) if custom.nil?
213
+
214
+ self.class.jitter(custom)
215
+ end
216
+
217
+ # The class whose retry hooks apply: the one an ActiveJob job's
218
+ # payload names in "wrapped", when it has them, else the job's own.
219
+ def hook_class(job_class, payload)
220
+ wrapped = payload["wrapped"]
221
+ return job_class unless wrapped.instance_of?(String)
222
+
223
+ klass = Object.const_get(wrapped)
224
+ klass.respond_to?(:robin_retry_in_block) ? klass : job_class
225
+ rescue NameError
226
+ job_class
227
+ end
228
+
229
+ # The robin_retry_in block's answer: seconds, :discard or :kill. Nil
230
+ # (use the default backoff) when it returns nil or false, raises, or
231
+ # returns something else; the last two are reported.
232
+ def custom_delay(block, failures, exception, payload)
233
+ custom = block.call(failures, exception, payload)
234
+ case custom
235
+ when nil, false then nil
236
+ when :discard, :kill, "discard", "kill" then custom.to_sym
237
+ else
238
+ return custom if custom.is_a?(Numeric) && custom.real? && custom.finite? && !custom.negative?
239
+
240
+ raise ArgumentError, "robin_retry_in returned #{custom.inspect}; expected seconds (>= 0), :discard, :kill or nil"
241
+ end
242
+ rescue => e
243
+ config.handle_exception(e, {context: "robin_retry_in block", job: payload})
244
+ nil
245
+ end
246
+
247
+ # The first +lines+ backtrace lines (after Config#backtrace_cleaner)
248
+ # as UTF-8; [] when there's no backtrace (the exception was never
249
+ # raised) or it can't be read.
250
+ def backtrace_of(exception, lines)
251
+ # (No backtrace (nil) raises NoMethodError here, rescued below.)
252
+ clean(exception.backtrace).first(lines).map { |line| Robin.utf8(line) } # steep:ignore NoMethod
253
+ rescue
254
+ []
255
+ end
256
+
257
+ # A cleaner that fails leaves the backtrace as it was.
258
+ def clean(backtrace)
259
+ cleaner = config.backtrace_cleaner
260
+ return backtrace unless cleaner && backtrace
261
+
262
+ cleaner.call(backtrace)
263
+ rescue => e
264
+ config.handle_exception(e, {context: "backtrace_cleaner"})
265
+ backtrace
266
+ end
267
+
268
+ def backtrace_lines(payload)
269
+ case (bt = payload["backtrace"])
270
+ when true then config.backtrace_lines
271
+ when Integer then bt
272
+ end
273
+ end
274
+
275
+ def truncate(message)
276
+ return message if message.bytesize <= MAX_MESSAGE_BYTES
277
+
278
+ # (byteslice within the message's bytes never returns nil.)
279
+ cut = message.byteslice(0, MAX_MESSAGE_BYTES) #: String
280
+ # Drops the character the cut may have split.
281
+ cut.scrub("") + "…"
282
+ end
283
+ end
284
+ end