falqon 0.0.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,640 @@
1
+ # frozen_string_literal: true
2
+
3
+ # typed: true
4
+
5
+ module Falqon
6
+ ##
7
+ # Simple, efficient, and reliable messaging queue implementation
8
+ #
9
+ class Queue
10
+ include Hooks
11
+ extend T::Sig
12
+
13
+ # The name of the queue (without prefix)
14
+ sig { returns(String) }
15
+ attr_reader :name
16
+
17
+ # The identifier of the queue (with prefix)
18
+ sig { returns(String) }
19
+ attr_reader :id
20
+
21
+ # The configured retry strategy of the queue
22
+ sig { returns(Strategy) }
23
+ attr_reader :retry_strategy
24
+
25
+ # The maximum number of retries before a message is considered failed
26
+ sig { returns(Integer) }
27
+ attr_reader :max_retries
28
+
29
+ # The delay in seconds before a message is eligible for a retry
30
+ sig { returns(Integer) }
31
+ attr_reader :retry_delay
32
+
33
+ # @!visibility private
34
+ sig { returns(ConnectionPool) }
35
+ attr_reader :redis
36
+
37
+ # @!visibility private
38
+ sig { returns(Logger) }
39
+ attr_reader :logger
40
+
41
+ # Create a new queue
42
+ #
43
+ # Create a new queue in Redis with the given name. If a queue with the same name already exists, it is reused.
44
+ # When registering a new queue, the following {Falqon::Queue::Metadata} is stored:
45
+ # - +created_at+: Timestamp of creation
46
+ # - +updated_at+: Timestamp of last update
47
+ # - +version+: Protocol version
48
+ #
49
+ # Initializing a queue with a different protocol version than the existing queue will raise a {Falqon::VersionMismatchError}.
50
+ # Currently queues are not compatible between different protocol versions, and must be deleted and recreated manually.
51
+ # In a future version, automatic migration between protocol versions may be supported.
52
+ #
53
+ # Please note that retry strategy, maximum retries, and retry delay are configured per queue instance, and are not shared between queue instances.
54
+ #
55
+ # @param name The name of the queue (without prefix)
56
+ # @param retry_strategy The retry strategy to use for failed messages
57
+ # @param max_retries The maximum number of retries before a message is considered failed
58
+ # @param retry_delay The delay in seconds before a message is eligible for a retry
59
+ # @param redis The Redis connection pool to use
60
+ # @param logger The logger to use
61
+ # @param version The protocol version to use
62
+ # @return The new queue
63
+ # @raise [Falqon::VersionMismatchError] if the protocol version of the existing queue does not match the protocol version of the new queue
64
+ #
65
+ # @example Create a new queue
66
+ # queue = Falqon::Queue.new("my_queue")
67
+ # queue.name # => "my_queue"
68
+ # queue.id # => "falqon/my_queue"
69
+ #
70
+ sig { params(name: String, retry_strategy: Symbol, max_retries: Integer, retry_delay: Integer, redis: ConnectionPool, logger: Logger, version: Integer).void }
71
+ def initialize(
72
+ name,
73
+ retry_strategy: Falqon.configuration.retry_strategy,
74
+ max_retries: Falqon.configuration.max_retries,
75
+ retry_delay: Falqon.configuration.retry_delay,
76
+ redis: Falqon.configuration.redis,
77
+ logger: Falqon.configuration.logger,
78
+ version: Falqon::PROTOCOL
79
+ )
80
+ @name = name
81
+ @id = [Falqon.configuration.prefix, name].compact.join("/")
82
+ @retry_strategy = Strategies.const_get(retry_strategy.to_s.capitalize).new(self)
83
+ @max_retries = max_retries
84
+ @retry_delay = retry_delay
85
+ @redis = redis
86
+ @logger = logger
87
+ @version = version
88
+
89
+ redis.with do |r|
90
+ queue_version = r.hget("#{id}:metadata", :version)
91
+
92
+ raise Falqon::VersionMismatchError, "Queue #{name} is using protocol version #{queue_version}, but this client is using protocol version #{version}" if queue_version && queue_version.to_i != @version
93
+
94
+ r.multi do |t|
95
+ # Register the queue
96
+ t.sadd([Falqon.configuration.prefix, "queues"].compact.join(":"), name)
97
+
98
+ # Set creation and update timestamp (if not set)
99
+ t.hsetnx("#{id}:metadata", :created_at, Time.now.to_i)
100
+ t.hsetnx("#{id}:metadata", :updated_at, Time.now.to_i)
101
+
102
+ # Set protocol version
103
+ t.hsetnx("#{id}:metadata", :version, @version)
104
+ end
105
+ end
106
+
107
+ run_hook :initialize, :after
108
+ end
109
+
110
+ # Push data onto the tail of the queue
111
+ #
112
+ # @param data The data to push onto the queue (one or more strings)
113
+ # @return The identifier(s) of the pushed message(s)
114
+ #
115
+ # @example Push a single message
116
+ # queue = Falqon::Queue.new("my_queue")
117
+ # queue.push("Hello, world!") # => "1"
118
+ #
119
+ # @example Push multiple messages
120
+ # queue = Falqon::Queue.new("my_queue")
121
+ # queue.push("Hello, world!", "Goodbye, world!") # => ["1", "2"]
122
+ #
123
+ sig { params(data: Data).returns(T.any(Identifier, T::Array[Identifier])) }
124
+ def push(*data)
125
+ logger.debug "Pushing #{data.size} messages onto queue #{name}"
126
+
127
+ run_hook :push, :before
128
+
129
+ # Set update timestamp
130
+ redis.with { |r| r.hset("#{id}:metadata", :updated_at, Time.now.to_i) }
131
+
132
+ ids = data.map do |d|
133
+ message = Message
134
+ .new(self, data: d)
135
+ .create
136
+
137
+ # Push identifier to queue
138
+ pending.add(message.id)
139
+
140
+ # Set message status
141
+ redis.with { |r| r.hset("#{id}:metadata:#{message.id}", :status, "pending") }
142
+
143
+ # Return identifier(s)
144
+ data.size == 1 ? (return message.id) : (next message.id)
145
+ end
146
+
147
+ run_hook :push, :after
148
+
149
+ # Return identifier(s)
150
+ ids
151
+ end
152
+
153
+ # Pop data from the head of the queue
154
+ #
155
+ # This method blocks until a message is available.
156
+ #
157
+ # == Acknowledgement
158
+ #
159
+ # If a block is given, the popped data is passed to the block. If the block raises a {Falqon::Error} exception, the message is retried according to the configured retry strategy.
160
+ # If no exception is raised, the message is ackwnowledged and removed from the queue.
161
+ #
162
+ # If no block is given, the popped data is returned.
163
+ # The message is immediately acknowledged and removed from the queue.
164
+ #
165
+ # @param block A block to execute with the popped data (block-style)
166
+ # @return The popped data (return-style)
167
+ #
168
+ # @example Pop a message (return-style)
169
+ # queue = Falqon::Queue.new("my_queue")
170
+ # queue.push("Hello, world!")
171
+ # queue.pop # => "Hello, world!"
172
+ #
173
+ # @example Pop a message (block-style)
174
+ # queue = Falqon::Queue.new("my_queue")
175
+ # queue.push("Hello, world!")
176
+ # queue.pop do |data|
177
+ # puts data # => "Hello, world!"
178
+ # end
179
+ #
180
+ sig { params(block: T.nilable(T.proc.params(data: Data).void)).returns(T.nilable(Data)) }
181
+ def pop(&block)
182
+ logger.debug "Popping message from queue #{name}"
183
+
184
+ run_hook :pop, :before
185
+
186
+ message = redis.with do |r|
187
+ # Move identifier from pending queue to processing queue
188
+ message_id = r.blmove(pending.id, processing.id, :left, :right).to_i
189
+
190
+ # Get retry count
191
+ retries = r.hget("#{id}:metadata:#{message_id}", :retries).to_i
192
+
193
+ r.multi do |t|
194
+ # Set message status
195
+ t.hset("#{id}:metadata:#{message_id}", :status, "processing")
196
+
197
+ # Set update timestamp
198
+ t.hset("#{id}:metadata", :updated_at, Time.now.to_i)
199
+ t.hset("#{id}:metadata:#{message_id}", :updated_at, Time.now.to_i)
200
+
201
+ # Increment processing counter
202
+ t.hincrby("#{id}:metadata", :processed, 1)
203
+
204
+ # Increment retry counter if message is retried
205
+ t.hincrby("#{id}:metadata", :retried, 1) if retries.positive?
206
+ end
207
+
208
+ Message.new(self, id: message_id)
209
+ end
210
+
211
+ data = message.data
212
+
213
+ yield data if block
214
+
215
+ run_hook :pop, :after
216
+
217
+ # Remove identifier from processing queue
218
+ processing.remove(message.id)
219
+
220
+ # Delete message
221
+ message.delete
222
+
223
+ data
224
+ rescue Error => e
225
+ logger.debug "Error processing message #{message.id}: #{e.message}"
226
+
227
+ # Increment failure counter
228
+ redis.with { |r| r.hincrby("#{id}:metadata", :failed, 1) }
229
+
230
+ # Retry message according to configured strategy
231
+ retry_strategy.retry(message, e)
232
+
233
+ nil
234
+ end
235
+
236
+ # Peek at the next message in the queue
237
+ #
238
+ # Use {#range} to peek at a range of messages.
239
+ # This method does not block.
240
+ #
241
+ # @param index The index of the message to peek at
242
+ # @return The data of the peeked message
243
+ #
244
+ # @example Peek at the next message
245
+ # queue = Falqon::Queue.new("my_queue")
246
+ # queue.push("Hello, world!")
247
+ # queue.peek # => "Hello, world!"
248
+ # queue.pop # => "Hello, world!"
249
+ #
250
+ # @example Peek at the next message with an offset
251
+ # queue = Falqon::Queue.new("my_queue")
252
+ # queue.push("Hello, world!", "Goodbye, world!")
253
+ # queue.peek(1) # => "Goodbye, world!"
254
+ # queue.pop # => "Hello, world!"
255
+ #
256
+ sig { params(index: Integer).returns(T.nilable(Data)) }
257
+ def peek(index: 0)
258
+ logger.debug "Peeking at next message in queue #{name}"
259
+
260
+ run_hook :peek, :before
261
+
262
+ # Get identifier from pending queue
263
+ message_id = pending.peek(index:)
264
+
265
+ return unless message_id
266
+
267
+ run_hook :peek, :after
268
+
269
+ # Retrieve data
270
+ Message.new(self, id: message_id).data
271
+ end
272
+
273
+ # Peek at the next messages in the queue
274
+ #
275
+ # Use {#peek} to peek at a single message.
276
+ # This method does not block.
277
+ #
278
+ # @param start The start index of the range to peek at
279
+ # @param stop The stop index of the range to peek at (set to -1 to peek at all messages)
280
+ # @return The data of the peeked messages
281
+ #
282
+ # @example Peek at the next messages
283
+ # queue = Falqon::Queue.new("my_queue")
284
+ # queue.push("Hello, world!", "Goodbye, world!", "Hello again, world!")
285
+ # queue.range(start: 1, stop: 2) # => ["Goodbye, world!", "Hello again, world!"]
286
+ # queue.range(start: 1) # => ["Goodbye, world!", "Hello again, world!"]
287
+ # queue.pop # => "Hello, world!"
288
+ #
289
+ sig { params(start: Integer, stop: Integer).returns(T::Array[Data]) }
290
+ def range(start: 0, stop: -1)
291
+ logger.debug "Peeking at next messages in queue #{name}"
292
+
293
+ run_hook :range, :before
294
+
295
+ # Get identifiers from pending queue
296
+ message_ids = pending.range(start:, stop:)
297
+
298
+ return [] unless message_ids.any?
299
+
300
+ run_hook :range, :after
301
+
302
+ # Retrieve data
303
+ message_ids.map { |id| Message.new(self, id:).data }
304
+ end
305
+
306
+ # Clear the queue, removing all messages
307
+ #
308
+ # This method clears all messages from the queue, including pending, processing, scheduled, and dead messages.
309
+ # It also resets the metadata counters for processed, failed, and retried messages, but does not deregister the queue.
310
+ #
311
+ # @return The identifiers of the cleared messages
312
+ #
313
+ # @example Clear the queue
314
+ # queue = Falqon::Queue.new("my_queue")
315
+ # queue.push("Hello, world!")
316
+ # queue.clear # => ["1"]
317
+ #
318
+ sig { returns(T::Array[Identifier]) }
319
+ def clear
320
+ logger.debug "Clearing queue #{name}"
321
+
322
+ run_hook :clear, :before
323
+
324
+ # Clear all sub-queues
325
+ message_ids = pending.clear + processing.clear + scheduled.clear + dead.clear
326
+
327
+ redis.with do |r|
328
+ r.multi do |t|
329
+ # Clear metadata
330
+ t.hdel("#{id}:metadata", :processed, :failed, :retried)
331
+
332
+ # Set update timestamp
333
+ t.hset("#{id}:metadata", :updated_at, Time.now.to_i)
334
+ end
335
+ end
336
+
337
+ run_hook :clear, :after
338
+
339
+ # Return identifiers
340
+ message_ids
341
+ end
342
+
343
+ # Delete the queue, removing all messages and deregistering the queue
344
+ #
345
+ # This method deletes the queue, removing all messages, metadata, and deregisters the queue.
346
+ #
347
+ # @example Delete the queue
348
+ # queue = Falqon::Queue.new("my_queue")
349
+ # queue.push("Hello, world!")
350
+ # queue.clear # => nil
351
+ #
352
+ sig { void }
353
+ def delete
354
+ logger.debug "Deleting queue #{name}"
355
+
356
+ run_hook :delete, :before
357
+
358
+ # Delete all sub-queues
359
+ [pending, processing, scheduled, dead]
360
+ .each(&:clear)
361
+
362
+ redis.with do |r|
363
+ r.multi do |t|
364
+ # Delete metadata
365
+ t.del("#{id}:metadata")
366
+
367
+ # Deregister the queue
368
+ t.srem([Falqon.configuration.prefix, "queues"].compact.join(":"), name)
369
+ end
370
+ end
371
+
372
+ run_hook :delete, :after
373
+ end
374
+
375
+ # Refill the queue with messages from the processing queue
376
+ #
377
+ # This method moves all messages from the processing queue back to the pending queue (in order).
378
+ # It is useful when a worker crashes or is stopped, and messages are left in the processing queue.
379
+ #
380
+ # @return The identifiers of the refilled messages
381
+ #
382
+ # @example Refill the queue
383
+ # queue = Falqon::Queue.new("my_queue")
384
+ # queue.push("Hello, world!")
385
+ # queue.pop { Kernel.exit! }
386
+ # ...
387
+ # queue.refill # => ["1"]
388
+ #
389
+ sig { returns(T::Array[Identifier]) }
390
+ def refill
391
+ logger.debug "Refilling queue #{name}"
392
+
393
+ run_hook :refill, :before
394
+
395
+ message_ids = []
396
+
397
+ # Move all identifiers from tail of processing queue to head of pending queue
398
+ redis.with do |r|
399
+ while (message_id = r.lmove(processing.id, id, :right, :left))
400
+ # Set message status
401
+ r.hset("#{id}:metadata:#{message_id}", :status, "pending")
402
+
403
+ message_ids << message_id
404
+ end
405
+ end
406
+
407
+ run_hook :refill, :after
408
+
409
+ message_ids
410
+ end
411
+
412
+ # Revive the queue with messages from the dead queue
413
+ #
414
+ # This method moves all messages from the dead queue back to the pending queue (in order).
415
+ # It is useful when messages are moved to the dead queue due to repeated failures, and need to be retried.
416
+ #
417
+ # @return The identifiers of the revived messages
418
+ #
419
+ # @example Revive the queue
420
+ # queue = Falqon::Queue.new("my_queue", max_retries: 0)
421
+ # queue.push("Hello, world!")
422
+ # queue.pop { raise Falqon::Error }
423
+ # queue.revive # => ["1"]
424
+ #
425
+ sig { returns(T::Array[Identifier]) }
426
+ def revive
427
+ logger.debug "Reviving queue #{name}"
428
+
429
+ run_hook :revive, :before
430
+
431
+ message_ids = []
432
+
433
+ # Move all identifiers from tail of dead queue to head of pending queue
434
+ redis.with do |r|
435
+ while (message_id = r.lmove(dead.id, id, :right, :left))
436
+ # Set message status
437
+ r.hset("#{id}:metadata:#{message_id}", :status, "pending")
438
+
439
+ message_ids << message_id
440
+ end
441
+ end
442
+
443
+ run_hook :revive, :after
444
+
445
+ message_ids
446
+ end
447
+
448
+ # Schedule failed messages for retry
449
+ #
450
+ # This method moves all eligible messages from the scheduled queue back to the head of the pending queue (in order).
451
+ # Messages are eligible for a retry according to the configured retry strategy.
452
+ #
453
+ # @return The identifiers of the scheduled messages
454
+ #
455
+ # @example Schedule failed messages
456
+ # queue = Falqon::Queue.new("my_queue", max_retries: 0, retry_delay: 5, retry_strategy: :linear)
457
+ # queue.push("Hello, world!")
458
+ # queue.pop { raise Falqon::Error }
459
+ # queue.schedule # => []
460
+ # sleep 5
461
+ # queue.schedule # => ["1"]
462
+ #
463
+ sig { returns(T::Array[Identifier]) }
464
+ def schedule
465
+ logger.debug "Scheduling failed messages on queue #{name}"
466
+
467
+ run_hook :schedule, :before
468
+
469
+ message_ids = T.let([], T::Array[Identifier])
470
+
471
+ # Move all due identifiers from scheduled queue to head of pending queue
472
+ redis.with do |r|
473
+ # Select all identifiers that are due (score <= current timestamp)
474
+ # FIXME: replace with zrange(by_score: true) when https://github.com/sds/mock_redis/issues/307 is resolved
475
+ # TODO: work in batches
476
+ message_ids = r.zrangebyscore(scheduled.id, 0, Time.now.to_i).map(&:to_i)
477
+
478
+ logger.debug "Scheduling messages #{message_ids.join(', ')} on queue #{name}"
479
+
480
+ r.multi do |t|
481
+ message_ids.each do |message_id|
482
+ # Set message status
483
+ t.hset("#{id}:metadata:#{message_id}", :status, "pending")
484
+
485
+ # Add identifier to pending queue
486
+ pending.add(message_id)
487
+
488
+ # Remove identifier from scheduled queue
489
+ scheduled.remove(message_id)
490
+ end
491
+ end
492
+ end
493
+
494
+ run_hook :schedule, :after
495
+
496
+ message_ids
497
+ end
498
+
499
+ # Size of the queue
500
+ #
501
+ # @return The number of messages in the queue
502
+ #
503
+ sig { returns(Integer) }
504
+ def size
505
+ pending.size
506
+ end
507
+
508
+ # Check if the queue is empty
509
+ #
510
+ # Only the pending queue is checked for emptiness.
511
+ #
512
+ # @return Whether the queue is empty
513
+ #
514
+ sig { returns(T::Boolean) }
515
+ def empty?
516
+ size.zero?
517
+ end
518
+
519
+ # Metadata of the queue
520
+ #
521
+ # @return The metadata of the queue
522
+ # @see Falqon::Queue::Metadata
523
+ #
524
+ sig { returns Metadata }
525
+ def metadata
526
+ redis.with do |r|
527
+ Metadata
528
+ .parse(r.hgetall("#{id}:metadata"))
529
+ end
530
+ end
531
+
532
+ # Subqueue for pending messages
533
+ # @!visibility private
534
+ #
535
+ # @return The subqueue for pending messages
536
+ #
537
+ sig { returns(SubQueue) }
538
+ def pending
539
+ @pending ||= SubQueue.new(self)
540
+ end
541
+
542
+ # Subqueue for processing messages
543
+ # @!visibility private
544
+ #
545
+ # @return The subqueue for processing messages
546
+ #
547
+ sig { returns(SubQueue) }
548
+ def processing
549
+ @processing ||= SubQueue.new(self, "processing")
550
+ end
551
+
552
+ # Subqueue for scheduled messages
553
+ # @!visibility private
554
+ #
555
+ # @return The subqueue for scheduled messages
556
+ #
557
+ sig { returns(SubSet) }
558
+ def scheduled
559
+ @scheduled ||= SubSet.new(self, "scheduled")
560
+ end
561
+
562
+ # Subqueue for dead messages
563
+ # @!visibility private
564
+ #
565
+ # @return The subqueue for dead messages
566
+ #
567
+ sig { returns(SubQueue) }
568
+ def dead
569
+ @dead ||= SubQueue.new(self, "dead")
570
+ end
571
+
572
+ # @!visibility private
573
+ sig { returns(String) }
574
+ def inspect
575
+ "#<#{self.class} name=#{name.inspect} pending=#{pending.size} processing=#{processing.size} scheduled=#{scheduled.size} dead=#{dead.size}>"
576
+ end
577
+
578
+ class << self
579
+ extend T::Sig
580
+ # Get a list of all registered queues
581
+ #
582
+ # @return The queues
583
+ #
584
+ sig { returns(T::Array[Queue]) }
585
+ def all
586
+ Falqon.configuration.redis.with do |r|
587
+ r
588
+ .smembers([Falqon.configuration.prefix, "queues"].compact.join(":"))
589
+ .map { |id| new(id) }
590
+ end
591
+ end
592
+
593
+ # Get the number of active (registered) queues
594
+ #
595
+ # @return The number of active (registered) queues
596
+ #
597
+ sig { returns(Integer) }
598
+ def size
599
+ Falqon.configuration.redis.with do |r|
600
+ r
601
+ .scard([Falqon.configuration.prefix, "queues"].compact.join(":"))
602
+ end
603
+ end
604
+ end
605
+
606
+ ##
607
+ # Queue metadata
608
+ #
609
+ class Metadata < T::Struct
610
+ # Total number of messages processed
611
+ prop :processed, Integer, default: 0
612
+
613
+ # Total number of messages failed
614
+ prop :failed, Integer, default: 0
615
+
616
+ # Total number of messages retried
617
+ prop :retried, Integer, default: 0
618
+
619
+ # Timestamp of creation
620
+ prop :created_at, Integer
621
+
622
+ # Timestamp of last update
623
+ prop :updated_at, Integer
624
+
625
+ # Protocol version
626
+ prop :version, Integer
627
+
628
+ # Parse metadata from Redis hash
629
+ #
630
+ # @!visibility private
631
+ #
632
+ def self.parse(data)
633
+ # Transform keys to symbols and values to integers
634
+ new data
635
+ .transform_keys(&:to_sym)
636
+ .transform_values(&:to_i)
637
+ end
638
+ end
639
+ end
640
+ end