falqon 0.0.1 → 1.0.0

data/lib/falqon/concerns/hooks.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  # Hooks can be registered on a custom queue to execute code before and after certain events.
+  # The following hooks are available:
+  # - +after :initialize+: executed after the queue has been initialized
+  # - +before :push+: executed before a message is pushed to the queue
+  # - +after :push+: executed after a message has been pushed to the queue
+  # - +before :pop+: executed before a message is popped from the queue
+  # - +after :pop+: executed after a message has been popped from the queue (but before deleting it)
+  # - +before :peek+: executed before peeking to a message in the queue
+  # - +after :peek+: executed after peeking to a message in the queue
+  # - +before :range+: executed before peeking to a message range in the queue
+  # - +after :range+: executed after peeking to a message range in the queue
+  # - +before :clear+: executed before clearing the queue
+  # - +after :clear+: executed after clearing the queue
+  # - +before :delete+: executed before deleting the queue
+  # - +after :delete+: executed after deleting the queue
+  # - +before :refill+: executed before refilling the queue
+  # - +after :refill+: executed after refilling the queue
+  # - +before :revive+: executed before reviving a message from the dead queue
+  # - +after :revive+: executed after reviving a message from the dead queue
+  # - +before :schedule+: executed before scheduling messages for retry
+  # - +after :schedule+: executed after scheduling messages for retry
+  #
+  # @example
+  #  class MyQueue < Falqon::Queue
+  #   before :push, :my_custom_method
+  #
+  #    after :delete do
+  #     ...
+  #    end
+  #
+  #    private
+  #
+  #    def my_custom_method
+  #      ...
+  #    end
+  #  end
+  #
+  module Hooks
+    extend T::Sig
+
+    # @!visibility private
+    sig { params(base: T.class_of(Hooks)).void }
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    # @!visibility private
+    sig { params(event: Symbol, type: T.nilable(Symbol), block: T.nilable(T.proc.void)).void }
+    def run_hook(event, type = nil, &block)
+      T.unsafe(self).class.hooks[event][:before].each { |hook| instance_eval(&hook) } if type.nil? || type == :before
+
+      block&.call
+
+      T.unsafe(self).class.hooks[event][:after].each { |hook| instance_eval(&hook) } if type.nil? || type == :after
+    end
+
+    module ClassMethods
+      include Kernel
+      extend T::Sig
+
+      # @!visibility private
+      sig { returns(T::Hash[Symbol, T::Hash[Symbol, T::Array[T.proc.void]]]) }
+      def hooks
+        @hooks ||= Hash.new { |h, k| h[k] = { before: [], after: [] } }
+      end
+
+      # Add hook to before list (either the given block or a wrapped method call)
+      #
+      # @param event The event to hook into
+      # @param method_sym The method to call
+      # @param block The block to execute
+      # @return [void]
+      #
+      sig { params(event: Symbol, method_sym: T.nilable(Symbol), block: T.nilable(T.proc.void)).void }
+      def before(event, method_sym = nil, &block)
+        block ||= proc { send(method_sym) } if method_sym
+
+        T.must(T.must(hooks[event])[:before]) << block if block
+      end
+
+      # Add hook to after list (either the given block or a wrapped method call)
+      #
+      # @param event The event to hook into
+      # @param method_sym The method to call
+      # @param block The block to execute
+      # @return [void]
+      #
+      sig { params(event: Symbol, method_sym: T.nilable(Symbol), block: T.nilable(T.proc.void)).void }
+      def after(event, method_sym = nil, &block)
+        block ||= proc { send(method_sym) } if method_sym
+
+        T.must(T.must(hooks[event])[:after]) << block if block
+      end
+    end
+  end
+end

data/lib/falqon/configuration.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+# typed: true
+
+require "logger"
+
+require "redis"
+require "connection_pool"
+
+module Falqon
+  # Falqon configuration
+  #
+  # Falqon can be configured before use, by leveraging the +Falqon.configure+ method.
+  # It's recommended to configure Falqon in an initializer file, such as +config/initializers/falqon.rb+.
+  # In a Rails application, the generator can be used to create the initializer file:
+  #
+  #   rails generate falqon:install
+  #
+  # Otherwise, the file can be created manually:
+  #
+  #   Falqon.configure do |config|
+  #     # Configure global queue name prefix
+  #     # config.prefix = ENV.fetch("FALQON_PREFIX", "falqon")
+  #
+  #     # Retry strategy (none or linear)
+  #     # config.retry_strategy = :linear
+  #
+  #     # Maximum number of retries before a message is discarded (-1 for infinite retries)
+  #     # config.max_retries = 3
+  #
+  #     # Retry delay (in seconds) for linear retry strategy (defaults to 0)
+  #     # config.retry_delay = 60
+  #
+  #     # Configure the Redis client options
+  #     # config.redis_options = { url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0") }
+  #
+  #     # Or, configure the Redis client directly
+  #     # config.redis = ConnectionPool.new(size: 5, timeout: 5) { Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) }
+  #
+  #     # Configure logger
+  #     # config.logger = Logger.new(STDOUT)
+  #   end
+  #
+  # The values above are the default values.
+  #
+  # In addition, it is recommended to configure Redis to be persistent in production environments, in order not to lose data.
+  # Refer to the {https://redis.io/docs/management/persistence Redis documentation} for more information.
+  #
+  class Configuration
+    extend T::Sig
+
+    # Queue name prefix
+    sig { params(prefix: String).returns(String) }
+    attr_writer :prefix
+
+    # Maximum number of retries before a message is discarded
+    sig { params(max_retries: Integer).returns(Integer) }
+    attr_writer :max_retries
+
+    # Delay between retries (in seconds)
+    sig { params(retry_delay: Integer).returns(Integer) }
+    attr_writer :retry_delay
+
+    # Redis connection pool
+    sig { params(redis: ConnectionPool).returns(ConnectionPool) }
+    attr_writer :redis
+
+    # Redis connection options
+    sig { params(redis_options: Hash).returns(Hash) }
+    attr_writer :redis_options
+
+    # Logger instance
+    sig { params(logger: Logger).returns(Logger) }
+    attr_writer :logger
+
+    # Queue name prefix, defaults to "falqon"
+    sig { returns(String) }
+    def prefix
+      @prefix ||= "falqon"
+    end
+
+    # Failed message retry strategy
+    #
+    # @see Falqon::Strategies
+    sig { returns(Symbol) }
+    def retry_strategy
+      @retry_strategy ||= :linear
+    end
+
+    # Failed message retry strategy
+    #
+    # @see Falqon::Strategies
+    sig { params(retry_strategy: Symbol).returns(Symbol) }
+    def retry_strategy=(retry_strategy)
+      raise ArgumentError, "Invalid retry strategy #{retry_strategy.inspect}" unless [:none, :linear].include? retry_strategy
+
+      @retry_strategy = retry_strategy
+    end
+
+    # Maximum number of retries before a message is discarded
+    #
+    # Only applicable when using the +:linear+ retry strategy
+    #
+    # @see Falqon::Strategies::Linear
+    sig { returns(Integer) }
+    def max_retries
+      @max_retries ||= 3
+    end
+
+    # Delay between retries (in seconds)
+    #
+    # Only applicable when using the +:linear+ retry strategy
+    #
+    # @see Falqon::Strategies::Linear
+    sig { returns(Integer) }
+    def retry_delay
+      @retry_delay ||= 0
+    end
+
+    # Redis connection pool
+    sig { returns(ConnectionPool) }
+    def redis
+      @redis ||= ConnectionPool.new(size: 5, timeout: 5) { Redis.new(**redis_options) }
+    end
+
+    # Redis connection options passed to +Redis.new+
+    sig { returns(Hash) }
+    def redis_options
+      @redis_options ||= {
+        url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0"),
+        middlewares: [Middlewares::Logger],
+      }
+    end
+
+    # Logger instance
+    sig { returns(Logger) }
+    def logger
+      @logger ||= Logger.new(File::NULL)
+    end
+  end
+end

data/lib/falqon/connection_pool_snooper.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Falqon
+  ##
+  # Connection pool that logs method calls
+  # @!visibility private
+  #
+  class ConnectionPoolSnooper < ConnectionPool
+    def with(...)
+      puts "#{caller(1..1).first[/.*:in/][0..-4]} #{caller(1..1).first[/`.*'/][1..-2]}"
+
+      super
+    end
+  end
+end

data/lib/falqon/data.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  extend T::Sig
+
+  # Base class for queue data
+  #
+  Data = T.type_alias { String }
+end

data/lib/falqon/error.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Falqon
+  # Base error class for Falqon
+  #
+  class Error < StandardError
+  end
+
+  # Error raised when a version mismatch is detected
+  #
+  class VersionMismatchError < Error
+  end
+end

data/lib/falqon/identifier.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  extend T::Sig
+
+  # Base class for queue identifiers
+  #
+  Identifier = T.type_alias { Integer }
+end

data/lib/falqon/message.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  ##
+  # A message in a queue
+  #
+  # This class should typically not be instantiated directly, but rather be created by a queue instance.
+  #
+  class Message
+    extend Forwardable
+    extend T::Sig
+
+    # The queue instance the message belongs to
+    sig { returns(Queue) }
+    attr_reader :queue
+
+    # Create a new message
+    #
+    # @param queue [Queue] The queue instance the message belongs to
+    # @param id [Integer] The message identifier (optional if creating a new message)
+    # @param data [String] The message data (optional if fetching an existing message)
+    # @return The message instance
+    #
+    # @example Instantiate an existing message
+    #   queue = Falqon::Queue.new("my_queue")
+    #   id = queue.push("Hello, World!")
+    #   message = Falqon::Message.new(queue, id:)
+    #   message.data # => "Hello, World!"
+    #
+    # @example Create a new message
+    #   queue = Falqon::Queue.new("my_queue")
+    #   message = Falqon::Message.new(queue, data: "Hello, World!")
+    #   message.create
+    #   message.id # => 1
+    #
+    sig { params(queue: Queue, id: T.nilable(Identifier), data: T.nilable(Data)).void }
+    def initialize(queue, id: nil, data: nil)
+      @queue = queue
+      @id = id
+      @data = data
+    end
+
+    # The message identifier
+    sig { returns(Identifier) }
+    def id
+      @id ||= redis.with { |r| r.incr("#{queue.id}:id") }
+    end
+
+    # The message data
+    sig { returns(String) }
+    def data
+      @data ||= redis.with { |r| r.get("#{queue.id}:data:#{id}") }
+    end
+
+    # Whether the message status is unknown
+    sig { returns(T::Boolean) }
+    def unknown?
+      metadata.status == "unknown"
+    end
+
+    # Whether the message status is pending
+    sig { returns(T::Boolean) }
+    def pending?
+      metadata.status == "pending"
+    end
+
+    # Whether the message status is processing
+    sig { returns(T::Boolean) }
+    def processing?
+      metadata.status == "processing"
+    end
+
+    # Whether the message status is scheduled
+    sig { returns(T::Boolean) }
+    def scheduled?
+      metadata.status == "scheduled"
+    end
+
+    # Whether the message status is dead
+    sig { returns(T::Boolean) }
+    def dead?
+      metadata.status == "dead"
+    end
+
+    # Whether the message exists (i.e. has been created)
+    sig { returns(T::Boolean) }
+    def exists?
+      redis.with do |r|
+        r.exists("#{queue.id}:data:#{id}") == 1
+      end
+    end
+
+    # Create the message in the queue
+    #
+    # This method will overwrite any existing message with the same identifier.
+    #
+    # @return The message instance
+    #
+    sig { returns(Message) }
+    def create
+      redis.with do |r|
+        message_id = id
+
+        r.multi do |t|
+          # Store data
+          t.set("#{queue.id}:data:#{message_id}", data)
+
+          # Set metadata
+          t.hset("#{queue.id}:metadata:#{message_id}",
+                 :created_at, Time.now.to_i,
+                 :updated_at, Time.now.to_i,)
+        end
+      end
+
+      self
+    end
+
+    # Kill the message
+    #
+    # This method moves the message to the dead queue, and resets the retry count.
+    #
+    sig { void }
+    def kill
+      logger.debug "Killing message #{id} on queue #{queue.name}"
+
+      redis.with do |r|
+        # Add identifier to dead queue
+        queue.dead.add(id)
+
+        # Reset retry count and set status to dead
+        r.hdel("#{queue.id}:metadata:#{id}", :retries)
+        r.hset("#{queue.id}:metadata:#{id}", :status, "dead")
+
+        # Remove identifier from queues
+        queue.pending.remove(id)
+      end
+    end
+
+    # Delete the message, removing it from the queue
+    #
+    # This method deletes the message, metadata, and data from the queue.
+    #
+    sig { void }
+    def delete
+      redis.with do |r|
+        r.multi do |t|
+          # Delete message from queue
+          queue.pending.remove(id)
+          queue.dead.remove(id)
+
+          # Delete data and metadata
+          t.del("#{queue.id}:data:#{id}", "#{queue.id}:metadata:#{id}")
+        end
+      end
+    end
+
+    # Message length
+    #
+    # @return The string length of the message (in bytes)
+    #
+    sig { returns Integer }
+    def size
+      redis.with { |r| r.strlen("#{queue.id}:data:#{id}") }
+    end
+
+    # Metadata of the message
+    #
+    # @return The metadata of the message
+    # @see Falqon::Message::Metadata
+    #
+    sig { returns Metadata }
+    def metadata
+      queue.redis.with do |r|
+        Metadata
+          .parse(r.hgetall("#{queue.id}:metadata:#{id}"))
+      end
+    end
+
+    sig { returns(String) }
+    def inspect
+      "#<#{self.class} id=#{id.inspect} size=#{size.inspect}>"
+    end
+
+    def_delegator :queue, :redis
+    def_delegator :queue, :logger
+
+    ##
+    # Metadata for a message
+    #
+    class Metadata < T::Struct
+      # Status (unknown, pending, processing, scheduled, dead)
+      prop :status, String, default: "unknown"
+
+      # Number of times the message has been retried
+      prop :retries, Integer, default: 0
+
+      # Timestamp of last retry
+      prop :retried_at, T.nilable(Integer)
+
+      # Last error message
+      prop :retry_error, T.nilable(String)
+
+      # Timestamp of creation
+      prop :created_at, Integer
+
+      # Timestamp of last update
+      prop :updated_at, Integer
+
+      # Parse metadata from Redis hash
+      #
+      # @!visibility private
+      #
+      def self.parse(data)
+        # Transform keys to symbols, and values to integers
+        new(data.to_h { |k, v| [k.to_sym, (send(props.dig(k.to_sym, :type).name.to_sym, v) if v)] })
+      end
+    end
+  end
+end

data/lib/falqon/middlewares/logger.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Falqon
+  # @!visibility private
+  module Middlewares
+    ##
+    # Redis client logger middleware
+    # @!visibility private
+    #
+    module Logger
+      def connect(redis_config)
+        Falqon.logger.warn { "[redis] #{redis_config.inspect}" }
+
+        super
+      end
+
+      def call(command, redis_config)
+        Falqon.logger.warn { "[redis] #{command.join(' ')}" }
+
+        super
+      end
+
+      def call_pipelined(commands, redis_config)
+        Falqon.logger.warn { "[redis] #{commands.join(' ')}" }
+
+        super
+      end
+    end
+  end
+end
+
+RedisClient.register(Falqon::Middlewares::Logger)