falqon 0.0.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +26 -1
- data/Gemfile +40 -8
- data/README.md +108 -8
- data/bin/falqon +8 -0
- data/config/inflections.rb +3 -1
- data/lib/falqon/cli/base.rb +35 -0
- data/lib/falqon/cli/clear.rb +86 -0
- data/lib/falqon/cli/delete.rb +152 -0
- data/lib/falqon/cli/kill.rb +143 -0
- data/lib/falqon/cli/list.rb +26 -0
- data/lib/falqon/cli/refill.rb +36 -0
- data/lib/falqon/cli/revive.rb +36 -0
- data/lib/falqon/cli/schedule.rb +40 -0
- data/lib/falqon/cli/show.rb +189 -0
- data/lib/falqon/cli/stats.rb +44 -0
- data/lib/falqon/cli/status.rb +47 -0
- data/lib/falqon/cli/version.rb +14 -0
- data/lib/falqon/cli.rb +168 -0
- data/lib/falqon/concerns/hooks.rb +101 -0
- data/lib/falqon/configuration.rb +141 -0
- data/lib/falqon/connection_pool_snooper.rb +15 -0
- data/lib/falqon/data.rb +11 -0
- data/lib/falqon/error.rb +13 -0
- data/lib/falqon/identifier.rb +11 -0
- data/lib/falqon/message.rb +221 -0
- data/lib/falqon/middlewares/logger.rb +32 -0
- data/lib/falqon/queue.rb +640 -0
- data/lib/falqon/strategies/linear.rb +82 -0
- data/lib/falqon/strategies/none.rb +44 -0
- data/lib/falqon/strategy.rb +26 -0
- data/lib/falqon/sub_queue.rb +96 -0
- data/lib/falqon/sub_set.rb +92 -0
- data/lib/falqon/version.rb +7 -2
- data/lib/falqon.rb +63 -0
- data/lib/generators/falqon/install.rb +37 -0
- metadata +100 -13
data/lib/falqon/queue.rb
ADDED
@@ -0,0 +1,640 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# typed: true
|
4
|
+
|
5
|
+
module Falqon
|
6
|
+
##
|
7
|
+
# Simple, efficient, and reliable messaging queue implementation
|
8
|
+
#
|
9
|
+
class Queue
|
10
|
+
include Hooks
|
11
|
+
extend T::Sig
|
12
|
+
|
13
|
+
# The name of the queue (without prefix)
|
14
|
+
sig { returns(String) }
|
15
|
+
attr_reader :name
|
16
|
+
|
17
|
+
# The identifier of the queue (with prefix)
|
18
|
+
sig { returns(String) }
|
19
|
+
attr_reader :id
|
20
|
+
|
21
|
+
# The configured retry strategy of the queue
|
22
|
+
sig { returns(Strategy) }
|
23
|
+
attr_reader :retry_strategy
|
24
|
+
|
25
|
+
# The maximum number of retries before a message is considered failed
|
26
|
+
sig { returns(Integer) }
|
27
|
+
attr_reader :max_retries
|
28
|
+
|
29
|
+
# The delay in seconds before a message is eligible for a retry
|
30
|
+
sig { returns(Integer) }
|
31
|
+
attr_reader :retry_delay
|
32
|
+
|
33
|
+
# @!visibility private
|
34
|
+
sig { returns(ConnectionPool) }
|
35
|
+
attr_reader :redis
|
36
|
+
|
37
|
+
# @!visibility private
|
38
|
+
sig { returns(Logger) }
|
39
|
+
attr_reader :logger
|
40
|
+
|
41
|
+
# Create a new queue
|
42
|
+
#
|
43
|
+
# Create a new queue in Redis with the given name. If a queue with the same name already exists, it is reused.
|
44
|
+
# When registering a new queue, the following {Falqon::Queue::Metadata} is stored:
|
45
|
+
# - +created_at+: Timestamp of creation
|
46
|
+
# - +updated_at+: Timestamp of last update
|
47
|
+
# - +version+: Protocol version
|
48
|
+
#
|
49
|
+
# Initializing a queue with a different protocol version than the existing queue will raise a {Falqon::VersionMismatchError}.
|
50
|
+
# Currently queues are not compatible between different protocol versions, and must be deleted and recreated manually.
|
51
|
+
# In a future version, automatic migration between protocol versions may be supported.
|
52
|
+
#
|
53
|
+
# Please note that retry strategy, maximum retries, and retry delay are configured per queue instance, and are not shared between queue instances.
|
54
|
+
#
|
55
|
+
# @param name The name of the queue (without prefix)
|
56
|
+
# @param retry_strategy The retry strategy to use for failed messages
|
57
|
+
# @param max_retries The maximum number of retries before a message is considered failed
|
58
|
+
# @param retry_delay The delay in seconds before a message is eligible for a retry
|
59
|
+
# @param redis The Redis connection pool to use
|
60
|
+
# @param logger The logger to use
|
61
|
+
# @param version The protocol version to use
|
62
|
+
# @return The new queue
|
63
|
+
# @raise [Falqon::VersionMismatchError] if the protocol version of the existing queue does not match the protocol version of the new queue
|
64
|
+
#
|
65
|
+
# @example Create a new queue
|
66
|
+
# queue = Falqon::Queue.new("my_queue")
|
67
|
+
# queue.name # => "my_queue"
|
68
|
+
# queue.id # => "falqon/my_queue"
|
69
|
+
#
|
70
|
+
sig { params(name: String, retry_strategy: Symbol, max_retries: Integer, retry_delay: Integer, redis: ConnectionPool, logger: Logger, version: Integer).void }
|
71
|
+
def initialize(
|
72
|
+
name,
|
73
|
+
retry_strategy: Falqon.configuration.retry_strategy,
|
74
|
+
max_retries: Falqon.configuration.max_retries,
|
75
|
+
retry_delay: Falqon.configuration.retry_delay,
|
76
|
+
redis: Falqon.configuration.redis,
|
77
|
+
logger: Falqon.configuration.logger,
|
78
|
+
version: Falqon::PROTOCOL
|
79
|
+
)
|
80
|
+
@name = name
|
81
|
+
@id = [Falqon.configuration.prefix, name].compact.join("/")
|
82
|
+
@retry_strategy = Strategies.const_get(retry_strategy.to_s.capitalize).new(self)
|
83
|
+
@max_retries = max_retries
|
84
|
+
@retry_delay = retry_delay
|
85
|
+
@redis = redis
|
86
|
+
@logger = logger
|
87
|
+
@version = version
|
88
|
+
|
89
|
+
redis.with do |r|
|
90
|
+
queue_version = r.hget("#{id}:metadata", :version)
|
91
|
+
|
92
|
+
raise Falqon::VersionMismatchError, "Queue #{name} is using protocol version #{queue_version}, but this client is using protocol version #{version}" if queue_version && queue_version.to_i != @version
|
93
|
+
|
94
|
+
r.multi do |t|
|
95
|
+
# Register the queue
|
96
|
+
t.sadd([Falqon.configuration.prefix, "queues"].compact.join(":"), name)
|
97
|
+
|
98
|
+
# Set creation and update timestamp (if not set)
|
99
|
+
t.hsetnx("#{id}:metadata", :created_at, Time.now.to_i)
|
100
|
+
t.hsetnx("#{id}:metadata", :updated_at, Time.now.to_i)
|
101
|
+
|
102
|
+
# Set protocol version
|
103
|
+
t.hsetnx("#{id}:metadata", :version, @version)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
run_hook :initialize, :after
|
108
|
+
end
|
109
|
+
|
110
|
+
# Push data onto the tail of the queue
|
111
|
+
#
|
112
|
+
# @param data The data to push onto the queue (one or more strings)
|
113
|
+
# @return The identifier(s) of the pushed message(s)
|
114
|
+
#
|
115
|
+
# @example Push a single message
|
116
|
+
# queue = Falqon::Queue.new("my_queue")
|
117
|
+
# queue.push("Hello, world!") # => "1"
|
118
|
+
#
|
119
|
+
# @example Push multiple messages
|
120
|
+
# queue = Falqon::Queue.new("my_queue")
|
121
|
+
# queue.push("Hello, world!", "Goodbye, world!") # => ["1", "2"]
|
122
|
+
#
|
123
|
+
sig { params(data: Data).returns(T.any(Identifier, T::Array[Identifier])) }
|
124
|
+
def push(*data)
|
125
|
+
logger.debug "Pushing #{data.size} messages onto queue #{name}"
|
126
|
+
|
127
|
+
run_hook :push, :before
|
128
|
+
|
129
|
+
# Set update timestamp
|
130
|
+
redis.with { |r| r.hset("#{id}:metadata", :updated_at, Time.now.to_i) }
|
131
|
+
|
132
|
+
ids = data.map do |d|
|
133
|
+
message = Message
|
134
|
+
.new(self, data: d)
|
135
|
+
.create
|
136
|
+
|
137
|
+
# Push identifier to queue
|
138
|
+
pending.add(message.id)
|
139
|
+
|
140
|
+
# Set message status
|
141
|
+
redis.with { |r| r.hset("#{id}:metadata:#{message.id}", :status, "pending") }
|
142
|
+
|
143
|
+
# Return identifier(s)
|
144
|
+
data.size == 1 ? (return message.id) : (next message.id)
|
145
|
+
end
|
146
|
+
|
147
|
+
run_hook :push, :after
|
148
|
+
|
149
|
+
# Return identifier(s)
|
150
|
+
ids
|
151
|
+
end
|
152
|
+
|
153
|
+
# Pop data from the head of the queue
|
154
|
+
#
|
155
|
+
# This method blocks until a message is available.
|
156
|
+
#
|
157
|
+
# == Acknowledgement
|
158
|
+
#
|
159
|
+
# If a block is given, the popped data is passed to the block. If the block raises a {Falqon::Error} exception, the message is retried according to the configured retry strategy.
|
160
|
+
# If no exception is raised, the message is ackwnowledged and removed from the queue.
|
161
|
+
#
|
162
|
+
# If no block is given, the popped data is returned.
|
163
|
+
# The message is immediately acknowledged and removed from the queue.
|
164
|
+
#
|
165
|
+
# @param block A block to execute with the popped data (block-style)
|
166
|
+
# @return The popped data (return-style)
|
167
|
+
#
|
168
|
+
# @example Pop a message (return-style)
|
169
|
+
# queue = Falqon::Queue.new("my_queue")
|
170
|
+
# queue.push("Hello, world!")
|
171
|
+
# queue.pop # => "Hello, world!"
|
172
|
+
#
|
173
|
+
# @example Pop a message (block-style)
|
174
|
+
# queue = Falqon::Queue.new("my_queue")
|
175
|
+
# queue.push("Hello, world!")
|
176
|
+
# queue.pop do |data|
|
177
|
+
# puts data # => "Hello, world!"
|
178
|
+
# end
|
179
|
+
#
|
180
|
+
sig { params(block: T.nilable(T.proc.params(data: Data).void)).returns(T.nilable(Data)) }
|
181
|
+
def pop(&block)
|
182
|
+
logger.debug "Popping message from queue #{name}"
|
183
|
+
|
184
|
+
run_hook :pop, :before
|
185
|
+
|
186
|
+
message = redis.with do |r|
|
187
|
+
# Move identifier from pending queue to processing queue
|
188
|
+
message_id = r.blmove(pending.id, processing.id, :left, :right).to_i
|
189
|
+
|
190
|
+
# Get retry count
|
191
|
+
retries = r.hget("#{id}:metadata:#{message_id}", :retries).to_i
|
192
|
+
|
193
|
+
r.multi do |t|
|
194
|
+
# Set message status
|
195
|
+
t.hset("#{id}:metadata:#{message_id}", :status, "processing")
|
196
|
+
|
197
|
+
# Set update timestamp
|
198
|
+
t.hset("#{id}:metadata", :updated_at, Time.now.to_i)
|
199
|
+
t.hset("#{id}:metadata:#{message_id}", :updated_at, Time.now.to_i)
|
200
|
+
|
201
|
+
# Increment processing counter
|
202
|
+
t.hincrby("#{id}:metadata", :processed, 1)
|
203
|
+
|
204
|
+
# Increment retry counter if message is retried
|
205
|
+
t.hincrby("#{id}:metadata", :retried, 1) if retries.positive?
|
206
|
+
end
|
207
|
+
|
208
|
+
Message.new(self, id: message_id)
|
209
|
+
end
|
210
|
+
|
211
|
+
data = message.data
|
212
|
+
|
213
|
+
yield data if block
|
214
|
+
|
215
|
+
run_hook :pop, :after
|
216
|
+
|
217
|
+
# Remove identifier from processing queue
|
218
|
+
processing.remove(message.id)
|
219
|
+
|
220
|
+
# Delete message
|
221
|
+
message.delete
|
222
|
+
|
223
|
+
data
|
224
|
+
rescue Error => e
|
225
|
+
logger.debug "Error processing message #{message.id}: #{e.message}"
|
226
|
+
|
227
|
+
# Increment failure counter
|
228
|
+
redis.with { |r| r.hincrby("#{id}:metadata", :failed, 1) }
|
229
|
+
|
230
|
+
# Retry message according to configured strategy
|
231
|
+
retry_strategy.retry(message, e)
|
232
|
+
|
233
|
+
nil
|
234
|
+
end
|
235
|
+
|
236
|
+
# Peek at the next message in the queue
|
237
|
+
#
|
238
|
+
# Use {#range} to peek at a range of messages.
|
239
|
+
# This method does not block.
|
240
|
+
#
|
241
|
+
# @param index The index of the message to peek at
|
242
|
+
# @return The data of the peeked message
|
243
|
+
#
|
244
|
+
# @example Peek at the next message
|
245
|
+
# queue = Falqon::Queue.new("my_queue")
|
246
|
+
# queue.push("Hello, world!")
|
247
|
+
# queue.peek # => "Hello, world!"
|
248
|
+
# queue.pop # => "Hello, world!"
|
249
|
+
#
|
250
|
+
# @example Peek at the next message with an offset
|
251
|
+
# queue = Falqon::Queue.new("my_queue")
|
252
|
+
# queue.push("Hello, world!", "Goodbye, world!")
|
253
|
+
# queue.peek(1) # => "Goodbye, world!"
|
254
|
+
# queue.pop # => "Hello, world!"
|
255
|
+
#
|
256
|
+
sig { params(index: Integer).returns(T.nilable(Data)) }
|
257
|
+
def peek(index: 0)
|
258
|
+
logger.debug "Peeking at next message in queue #{name}"
|
259
|
+
|
260
|
+
run_hook :peek, :before
|
261
|
+
|
262
|
+
# Get identifier from pending queue
|
263
|
+
message_id = pending.peek(index:)
|
264
|
+
|
265
|
+
return unless message_id
|
266
|
+
|
267
|
+
run_hook :peek, :after
|
268
|
+
|
269
|
+
# Retrieve data
|
270
|
+
Message.new(self, id: message_id).data
|
271
|
+
end
|
272
|
+
|
273
|
+
# Peek at the next messages in the queue
|
274
|
+
#
|
275
|
+
# Use {#peek} to peek at a single message.
|
276
|
+
# This method does not block.
|
277
|
+
#
|
278
|
+
# @param start The start index of the range to peek at
|
279
|
+
# @param stop The stop index of the range to peek at (set to -1 to peek at all messages)
|
280
|
+
# @return The data of the peeked messages
|
281
|
+
#
|
282
|
+
# @example Peek at the next messages
|
283
|
+
# queue = Falqon::Queue.new("my_queue")
|
284
|
+
# queue.push("Hello, world!", "Goodbye, world!", "Hello again, world!")
|
285
|
+
# queue.range(start: 1, stop: 2) # => ["Goodbye, world!", "Hello again, world!"]
|
286
|
+
# queue.range(start: 1) # => ["Goodbye, world!", "Hello again, world!"]
|
287
|
+
# queue.pop # => "Hello, world!"
|
288
|
+
#
|
289
|
+
sig { params(start: Integer, stop: Integer).returns(T::Array[Data]) }
|
290
|
+
def range(start: 0, stop: -1)
|
291
|
+
logger.debug "Peeking at next messages in queue #{name}"
|
292
|
+
|
293
|
+
run_hook :range, :before
|
294
|
+
|
295
|
+
# Get identifiers from pending queue
|
296
|
+
message_ids = pending.range(start:, stop:)
|
297
|
+
|
298
|
+
return [] unless message_ids.any?
|
299
|
+
|
300
|
+
run_hook :range, :after
|
301
|
+
|
302
|
+
# Retrieve data
|
303
|
+
message_ids.map { |id| Message.new(self, id:).data }
|
304
|
+
end
|
305
|
+
|
306
|
+
# Clear the queue, removing all messages
|
307
|
+
#
|
308
|
+
# This method clears all messages from the queue, including pending, processing, scheduled, and dead messages.
|
309
|
+
# It also resets the metadata counters for processed, failed, and retried messages, but does not deregister the queue.
|
310
|
+
#
|
311
|
+
# @return The identifiers of the cleared messages
|
312
|
+
#
|
313
|
+
# @example Clear the queue
|
314
|
+
# queue = Falqon::Queue.new("my_queue")
|
315
|
+
# queue.push("Hello, world!")
|
316
|
+
# queue.clear # => ["1"]
|
317
|
+
#
|
318
|
+
sig { returns(T::Array[Identifier]) }
|
319
|
+
def clear
|
320
|
+
logger.debug "Clearing queue #{name}"
|
321
|
+
|
322
|
+
run_hook :clear, :before
|
323
|
+
|
324
|
+
# Clear all sub-queues
|
325
|
+
message_ids = pending.clear + processing.clear + scheduled.clear + dead.clear
|
326
|
+
|
327
|
+
redis.with do |r|
|
328
|
+
r.multi do |t|
|
329
|
+
# Clear metadata
|
330
|
+
t.hdel("#{id}:metadata", :processed, :failed, :retried)
|
331
|
+
|
332
|
+
# Set update timestamp
|
333
|
+
t.hset("#{id}:metadata", :updated_at, Time.now.to_i)
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
run_hook :clear, :after
|
338
|
+
|
339
|
+
# Return identifiers
|
340
|
+
message_ids
|
341
|
+
end
|
342
|
+
|
343
|
+
# Delete the queue, removing all messages and deregistering the queue
|
344
|
+
#
|
345
|
+
# This method deletes the queue, removing all messages, metadata, and deregisters the queue.
|
346
|
+
#
|
347
|
+
# @example Delete the queue
|
348
|
+
# queue = Falqon::Queue.new("my_queue")
|
349
|
+
# queue.push("Hello, world!")
|
350
|
+
# queue.clear # => nil
|
351
|
+
#
|
352
|
+
sig { void }
|
353
|
+
def delete
|
354
|
+
logger.debug "Deleting queue #{name}"
|
355
|
+
|
356
|
+
run_hook :delete, :before
|
357
|
+
|
358
|
+
# Delete all sub-queues
|
359
|
+
[pending, processing, scheduled, dead]
|
360
|
+
.each(&:clear)
|
361
|
+
|
362
|
+
redis.with do |r|
|
363
|
+
r.multi do |t|
|
364
|
+
# Delete metadata
|
365
|
+
t.del("#{id}:metadata")
|
366
|
+
|
367
|
+
# Deregister the queue
|
368
|
+
t.srem([Falqon.configuration.prefix, "queues"].compact.join(":"), name)
|
369
|
+
end
|
370
|
+
end
|
371
|
+
|
372
|
+
run_hook :delete, :after
|
373
|
+
end
|
374
|
+
|
375
|
+
# Refill the queue with messages from the processing queue
|
376
|
+
#
|
377
|
+
# This method moves all messages from the processing queue back to the pending queue (in order).
|
378
|
+
# It is useful when a worker crashes or is stopped, and messages are left in the processing queue.
|
379
|
+
#
|
380
|
+
# @return The identifiers of the refilled messages
|
381
|
+
#
|
382
|
+
# @example Refill the queue
|
383
|
+
# queue = Falqon::Queue.new("my_queue")
|
384
|
+
# queue.push("Hello, world!")
|
385
|
+
# queue.pop { Kernel.exit! }
|
386
|
+
# ...
|
387
|
+
# queue.refill # => ["1"]
|
388
|
+
#
|
389
|
+
sig { returns(T::Array[Identifier]) }
|
390
|
+
def refill
|
391
|
+
logger.debug "Refilling queue #{name}"
|
392
|
+
|
393
|
+
run_hook :refill, :before
|
394
|
+
|
395
|
+
message_ids = []
|
396
|
+
|
397
|
+
# Move all identifiers from tail of processing queue to head of pending queue
|
398
|
+
redis.with do |r|
|
399
|
+
while (message_id = r.lmove(processing.id, id, :right, :left))
|
400
|
+
# Set message status
|
401
|
+
r.hset("#{id}:metadata:#{message_id}", :status, "pending")
|
402
|
+
|
403
|
+
message_ids << message_id
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
run_hook :refill, :after
|
408
|
+
|
409
|
+
message_ids
|
410
|
+
end
|
411
|
+
|
412
|
+
# Revive the queue with messages from the dead queue
|
413
|
+
#
|
414
|
+
# This method moves all messages from the dead queue back to the pending queue (in order).
|
415
|
+
# It is useful when messages are moved to the dead queue due to repeated failures, and need to be retried.
|
416
|
+
#
|
417
|
+
# @return The identifiers of the revived messages
|
418
|
+
#
|
419
|
+
# @example Revive the queue
|
420
|
+
# queue = Falqon::Queue.new("my_queue", max_retries: 0)
|
421
|
+
# queue.push("Hello, world!")
|
422
|
+
# queue.pop { raise Falqon::Error }
|
423
|
+
# queue.revive # => ["1"]
|
424
|
+
#
|
425
|
+
sig { returns(T::Array[Identifier]) }
|
426
|
+
def revive
|
427
|
+
logger.debug "Reviving queue #{name}"
|
428
|
+
|
429
|
+
run_hook :revive, :before
|
430
|
+
|
431
|
+
message_ids = []
|
432
|
+
|
433
|
+
# Move all identifiers from tail of dead queue to head of pending queue
|
434
|
+
redis.with do |r|
|
435
|
+
while (message_id = r.lmove(dead.id, id, :right, :left))
|
436
|
+
# Set message status
|
437
|
+
r.hset("#{id}:metadata:#{message_id}", :status, "pending")
|
438
|
+
|
439
|
+
message_ids << message_id
|
440
|
+
end
|
441
|
+
end
|
442
|
+
|
443
|
+
run_hook :revive, :after
|
444
|
+
|
445
|
+
message_ids
|
446
|
+
end
|
447
|
+
|
448
|
+
# Schedule failed messages for retry
|
449
|
+
#
|
450
|
+
# This method moves all eligible messages from the scheduled queue back to the head of the pending queue (in order).
|
451
|
+
# Messages are eligible for a retry according to the configured retry strategy.
|
452
|
+
#
|
453
|
+
# @return The identifiers of the scheduled messages
|
454
|
+
#
|
455
|
+
# @example Schedule failed messages
|
456
|
+
# queue = Falqon::Queue.new("my_queue", max_retries: 0, retry_delay: 5, retry_strategy: :linear)
|
457
|
+
# queue.push("Hello, world!")
|
458
|
+
# queue.pop { raise Falqon::Error }
|
459
|
+
# queue.schedule # => []
|
460
|
+
# sleep 5
|
461
|
+
# queue.schedule # => ["1"]
|
462
|
+
#
|
463
|
+
sig { returns(T::Array[Identifier]) }
|
464
|
+
def schedule
|
465
|
+
logger.debug "Scheduling failed messages on queue #{name}"
|
466
|
+
|
467
|
+
run_hook :schedule, :before
|
468
|
+
|
469
|
+
message_ids = T.let([], T::Array[Identifier])
|
470
|
+
|
471
|
+
# Move all due identifiers from scheduled queue to head of pending queue
|
472
|
+
redis.with do |r|
|
473
|
+
# Select all identifiers that are due (score <= current timestamp)
|
474
|
+
# FIXME: replace with zrange(by_score: true) when https://github.com/sds/mock_redis/issues/307 is resolved
|
475
|
+
# TODO: work in batches
|
476
|
+
message_ids = r.zrangebyscore(scheduled.id, 0, Time.now.to_i).map(&:to_i)
|
477
|
+
|
478
|
+
logger.debug "Scheduling messages #{message_ids.join(', ')} on queue #{name}"
|
479
|
+
|
480
|
+
r.multi do |t|
|
481
|
+
message_ids.each do |message_id|
|
482
|
+
# Set message status
|
483
|
+
t.hset("#{id}:metadata:#{message_id}", :status, "pending")
|
484
|
+
|
485
|
+
# Add identifier to pending queue
|
486
|
+
pending.add(message_id)
|
487
|
+
|
488
|
+
# Remove identifier from scheduled queue
|
489
|
+
scheduled.remove(message_id)
|
490
|
+
end
|
491
|
+
end
|
492
|
+
end
|
493
|
+
|
494
|
+
run_hook :schedule, :after
|
495
|
+
|
496
|
+
message_ids
|
497
|
+
end
|
498
|
+
|
499
|
+
# Size of the queue
|
500
|
+
#
|
501
|
+
# @return The number of messages in the queue
|
502
|
+
#
|
503
|
+
sig { returns(Integer) }
|
504
|
+
def size
|
505
|
+
pending.size
|
506
|
+
end
|
507
|
+
|
508
|
+
# Check if the queue is empty
|
509
|
+
#
|
510
|
+
# Only the pending queue is checked for emptiness.
|
511
|
+
#
|
512
|
+
# @return Whether the queue is empty
|
513
|
+
#
|
514
|
+
sig { returns(T::Boolean) }
|
515
|
+
def empty?
|
516
|
+
size.zero?
|
517
|
+
end
|
518
|
+
|
519
|
+
# Metadata of the queue
|
520
|
+
#
|
521
|
+
# @return The metadata of the queue
|
522
|
+
# @see Falqon::Queue::Metadata
|
523
|
+
#
|
524
|
+
sig { returns Metadata }
|
525
|
+
def metadata
|
526
|
+
redis.with do |r|
|
527
|
+
Metadata
|
528
|
+
.parse(r.hgetall("#{id}:metadata"))
|
529
|
+
end
|
530
|
+
end
|
531
|
+
|
532
|
+
# Subqueue for pending messages
|
533
|
+
# @!visibility private
|
534
|
+
#
|
535
|
+
# @return The subqueue for pending messages
|
536
|
+
#
|
537
|
+
sig { returns(SubQueue) }
|
538
|
+
def pending
|
539
|
+
@pending ||= SubQueue.new(self)
|
540
|
+
end
|
541
|
+
|
542
|
+
# Subqueue for processing messages
|
543
|
+
# @!visibility private
|
544
|
+
#
|
545
|
+
# @return The subqueue for processing messages
|
546
|
+
#
|
547
|
+
sig { returns(SubQueue) }
|
548
|
+
def processing
|
549
|
+
@processing ||= SubQueue.new(self, "processing")
|
550
|
+
end
|
551
|
+
|
552
|
+
# Subqueue for scheduled messages
|
553
|
+
# @!visibility private
|
554
|
+
#
|
555
|
+
# @return The subqueue for scheduled messages
|
556
|
+
#
|
557
|
+
sig { returns(SubSet) }
|
558
|
+
def scheduled
|
559
|
+
@scheduled ||= SubSet.new(self, "scheduled")
|
560
|
+
end
|
561
|
+
|
562
|
+
# Subqueue for dead messages
|
563
|
+
# @!visibility private
|
564
|
+
#
|
565
|
+
# @return The subqueue for dead messages
|
566
|
+
#
|
567
|
+
sig { returns(SubQueue) }
|
568
|
+
def dead
|
569
|
+
@dead ||= SubQueue.new(self, "dead")
|
570
|
+
end
|
571
|
+
|
572
|
+
# @!visibility private
|
573
|
+
sig { returns(String) }
|
574
|
+
def inspect
|
575
|
+
"#<#{self.class} name=#{name.inspect} pending=#{pending.size} processing=#{processing.size} scheduled=#{scheduled.size} dead=#{dead.size}>"
|
576
|
+
end
|
577
|
+
|
578
|
+
class << self
|
579
|
+
extend T::Sig
|
580
|
+
# Get a list of all registered queues
|
581
|
+
#
|
582
|
+
# @return The queues
|
583
|
+
#
|
584
|
+
sig { returns(T::Array[Queue]) }
|
585
|
+
def all
|
586
|
+
Falqon.configuration.redis.with do |r|
|
587
|
+
r
|
588
|
+
.smembers([Falqon.configuration.prefix, "queues"].compact.join(":"))
|
589
|
+
.map { |id| new(id) }
|
590
|
+
end
|
591
|
+
end
|
592
|
+
|
593
|
+
# Get the number of active (registered) queues
|
594
|
+
#
|
595
|
+
# @return The number of active (registered) queues
|
596
|
+
#
|
597
|
+
sig { returns(Integer) }
|
598
|
+
def size
|
599
|
+
Falqon.configuration.redis.with do |r|
|
600
|
+
r
|
601
|
+
.scard([Falqon.configuration.prefix, "queues"].compact.join(":"))
|
602
|
+
end
|
603
|
+
end
|
604
|
+
end
|
605
|
+
|
606
|
+
##
|
607
|
+
# Queue metadata
|
608
|
+
#
|
609
|
+
class Metadata < T::Struct
|
610
|
+
# Total number of messages processed
|
611
|
+
prop :processed, Integer, default: 0
|
612
|
+
|
613
|
+
# Total number of messages failed
|
614
|
+
prop :failed, Integer, default: 0
|
615
|
+
|
616
|
+
# Total number of messages retried
|
617
|
+
prop :retried, Integer, default: 0
|
618
|
+
|
619
|
+
# Timestamp of creation
|
620
|
+
prop :created_at, Integer
|
621
|
+
|
622
|
+
# Timestamp of last update
|
623
|
+
prop :updated_at, Integer
|
624
|
+
|
625
|
+
# Protocol version
|
626
|
+
prop :version, Integer
|
627
|
+
|
628
|
+
# Parse metadata from Redis hash
|
629
|
+
#
|
630
|
+
# @!visibility private
|
631
|
+
#
|
632
|
+
def self.parse(data)
|
633
|
+
# Transform keys to symbols and values to integers
|
634
|
+
new data
|
635
|
+
.transform_keys(&:to_sym)
|
636
|
+
.transform_values(&:to_i)
|
637
|
+
end
|
638
|
+
end
|
639
|
+
end
|
640
|
+
end
|