falqon 0.0.1 → 1.0.0

data/lib/falqon/strategies/linear.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  module Strategies
+    ##
+    # Retry strategy that retries a fixed number of times
+    #
+    # When a message fails to process, it is moved to the scheduled queue, and retried after a fixed delay (configured by {Falqon::Queue#retry_delay}).
+    # If a messages fails to process after the maximum number of retries (configured by {Falqon::Queue#max_retries}), it is marked as dead, and moved to the dead subqueue.
+    #
+    # When using the linear strategy and the retry delay is set to a non-zero value, a scheduled needs to be started to retry the messages after the configured delay.
+    #
+    #  queue = Falqon::Queue.new("my_queue")
+    #
+    #  # Start the watcher in a separate thread
+    #  Thread.new { loop { queue.schedule; sleep 1 } }
+    #
+    #  # Or start the watcher in a separate fiber
+    #  Fiber
+    #    .new { loop { queue.schedule; sleep 1 } }
+    #    .resume
+    #
+    # @example
+    #  queue = Falqon::Queue.new("my_queue", retry_strategy: :linear, retry_delay: 60, max_retries: 3)
+    #  queue.push("Hello, World!")
+    #  queue.pop { raise Falqon::Error }
+    #  queue.inspect # => #<Falqon::Queue name="my_queue" pending=0 processing=0 scheduled=1 dead=0>
+    #  sleep 60
+    #  queue.pop # => "Hello, World!"
+    #
+    class Linear < Strategy
+      # @!visibility private
+      sig { params(message: Message, error: Error).void }
+      def retry(message, error)
+        queue.redis.with do |r|
+          # Increment retry count
+          retries = r.hincrby("#{queue.id}:metadata:#{message.id}", :retries, 1)
+
+          r.multi do |t|
+            # Set error metadata
+            t.hset(
+              "#{queue.id}:metadata:#{message.id}",
+              :retried_at, Time.now.to_i,
+              :retry_error, error.message,
+            )
+
+            if retries < queue.max_retries || queue.max_retries == -1
+              if queue.retry_delay.positive?
+                queue.logger.debug "Scheduling message #{message.id} on queue #{queue.name} in #{queue.retry_delay} seconds (attempt #{retries})"
+
+                # Add identifier to scheduled queue
+                queue.scheduled.add(message.id, Time.now.to_i + queue.retry_delay)
+
+                # Set message status
+                t.hset("#{queue.id}:metadata:#{message.id}", :status, "scheduled")
+              else
+                queue.logger.debug "Requeuing message #{message.id} on queue #{queue.name} (attempt #{retries})"
+
+                # Add identifier back to pending queue
+                queue.pending.add(message.id)
+
+                # Set message status
+                t.hset("#{queue.id}:metadata:#{message.id}", :status, "pending")
+              end
+            else
+              # Kill message after max retries
+              message.kill
+
+              # Set message status
+              t.hset("#{queue.id}:metadata:#{message.id}", :status, "dead")
+            end
+
+            # Remove identifier from processing queue
+            queue.processing.remove(message.id)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/falqon/strategies/none.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  module Strategies
+    ##
+    # Retry strategy that does not retry
+    #
+    # When a message fails to process, it is immediately marked as dead and moved to the dead subqueue.
+    #
+    # @example
+    #  queue = Falqon::Queue.new("my_queue", retry_strategy: :none)
+    #  queue.push("Hello, World!")
+    #  queue.pop { raise Falqon::Error }
+    #  queue.inspect # => #<Falqon::Queue name="my_queue" pending=0 processing=0 scheduled=0 dead=1>
+    #
+    class None < Strategy
+      # @!visibility private
+      sig { params(message: Message, error: Error).void }
+      def retry(message, error)
+        queue.redis.with do |r|
+          r.multi do |t|
+            # Set error metadata
+            t.hset(
+              "#{queue.id}:metadata:#{message.id}",
+              :retried_at, Time.now.to_i,
+              :retry_error, error.message,
+            )
+
+            # Kill message immediately
+            message.kill
+
+            # Remove identifier from processing queue
+            queue.processing.remove(message.id)
+
+            # Set message status
+            t.hset("#{queue.id}:metadata:#{message.id}", :status, "dead")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/falqon/strategy.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  ##
+  # Base class for retry strategies
+  # @!visibility private
+  #
+  class Strategy
+    extend T::Sig
+
+    sig { returns(Queue) }
+    attr_reader :queue
+
+    sig { params(queue: Queue).void }
+    def initialize(queue)
+      @queue = queue
+    end
+
+    sig { params(message: Message, error: Error).void }
+    def retry(message, error)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/falqon/sub_queue.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  ##
+  # Simple queue abstraction on top of Redis
+  # @!visibility private
+  #
+  class SubQueue
+    extend T::Sig
+
+    sig { returns(String) }
+    attr_reader :type
+
+    sig { returns(String) }
+    attr_reader :id
+
+    sig { returns(Queue) }
+    attr_reader :queue
+
+    sig { params(queue: Queue, type: T.nilable(String)).void }
+    def initialize(queue, type = nil)
+      @type = type || "pending"
+      @id = [queue.id, type].compact.join(":")
+      @queue = queue
+    end
+
+    sig { params(message_id: Identifier, head: T.nilable(T::Boolean)).void }
+    def add(message_id, head: false)
+      queue.redis.with do |r|
+        if head
+          r.lpush(id, message_id)
+        else
+          r.rpush(id, message_id)
+        end
+      end
+    end
+
+    sig { params(message_id: Identifier).void }
+    def remove(message_id)
+      queue.redis.with do |r|
+        r.lrem(id, 0, message_id)
+      end
+    end
+
+    sig { params(index: Integer).returns(T.nilable(Identifier)) }
+    def peek(index: 0)
+      queue.redis.with do |r|
+        r.lindex(id, index)&.to_i
+      end
+    end
+
+    sig { params(start: Integer, stop: Integer).returns(T::Array[Identifier]) }
+    def range(start: 0, stop: -1)
+      queue.redis.with do |r|
+        r.lrange(id, start, stop).map(&:to_i)
+      end
+    end
+
+    sig { returns(T::Array[Identifier]) }
+    def clear
+      queue.redis.with do |r|
+        # Get all identifiers from queue
+        message_ids = r.lrange(id, 0, -1)
+
+        # Delete all data and clear queue
+        # TODO: clear data in batches
+        r.del(*message_ids.flat_map { |message_id| ["#{queue.id}:data:#{message_id}", "#{queue.id}:metadata:#{message_id}"] }, id, "#{queue.id}:id")
+
+        # Return identifiers
+        message_ids.map(&:to_i)
+      end
+    end
+
+    sig { returns(Integer) }
+    def size
+      queue.redis.with { |r| r.llen(id) }
+    end
+
+    sig { returns(T::Boolean) }
+    def empty?
+      size.zero?
+    end
+
+    sig { returns(T::Array[Identifier]) }
+    def to_a
+      queue.redis.with { |r| r.lrange(id, 0, -1).map(&:to_i) }
+    end
+
+    sig { returns(String) }
+    def inspect
+      "#<#{self.class} name=#{type} size=#{size}>"
+    end
+  end
+end

data/lib/falqon/sub_set.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# typed: true
+
+module Falqon
+  ##
+  # Simple sorted set abstraction on top of Redis
+  # @!visibility private
+  #
+  class SubSet
+    extend T::Sig
+
+    sig { returns(String) }
+    attr_reader :type
+
+    sig { returns(String) }
+    attr_reader :id
+
+    sig { returns(Queue) }
+    attr_reader :queue
+
+    sig { params(queue: Queue, type: T.nilable(String)).void }
+    def initialize(queue, type = nil)
+      @type = type || "pending"
+      @id = [queue.id, type].compact.join(":")
+      @queue = queue
+    end
+
+    sig { params(message_id: Identifier, score: Integer).void }
+    def add(message_id, score)
+      queue.redis.with do |r|
+        r.zadd(id, score, message_id)
+      end
+    end
+
+    sig { params(message_id: Identifier).void }
+    def remove(message_id)
+      queue.redis.with do |r|
+        r.zrem(id, message_id)
+      end
+    end
+
+    sig { params(index: Integer).returns(T.nilable(Identifier)) }
+    def peek(index: 0)
+      queue.redis.with do |r|
+        r.zrange(id, index, index).first&.to_i
+      end
+    end
+
+    sig { params(start: Integer, stop: Integer).returns(T::Array[Identifier]) }
+    def range(start: 0, stop: -1)
+      queue.redis.with do |r|
+        r.zrange(id, start, stop).map(&:to_i)
+      end
+    end
+
+    sig { returns(T::Array[Identifier]) }
+    def clear
+      queue.redis.with do |r|
+        # Get all identifiers from queue
+        # TODO: work in batches
+        message_ids = r.zrange(id, 0, -1)
+
+        # Delete all data and clear queue
+        r.del(*message_ids.flat_map { |message_id| ["#{queue.id}:data:#{message_id}", "#{queue.id}:metadata:#{message_id}"] }, id, "#{queue.id}:id")
+
+        # Return identifiers
+        message_ids.map(&:to_i)
+      end
+    end
+
+    sig { returns(Integer) }
+    def size
+      queue.redis.with { |r| r.zcount(id, "-inf", "+inf") }
+    end
+
+    sig { returns(T::Boolean) }
+    def empty?
+      size.zero?
+    end
+
+    sig { returns(T::Array[Identifier]) }
+    def to_a
+      queue.redis.with { |r| r.zrange(id, 0, -1).map(&:to_i) }
+    end
+
+    sig { returns(String) }
+    def inspect
+      "#<#{self.class} name=#{type} size=#{size}>"
+    end
+  end
+end

data/lib/falqon/version.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 module Falqon
+  # @!visibility private
   module Version
-    MAJOR = 0
+    MAJOR = 1
     MINOR = 0
-    PATCH = 1
+    PATCH = 0
     PRE   = nil
 
     VERSION = [MAJOR, MINOR, PATCH].compact.join(".")
@@ -12,5 +13,9 @@ module Falqon
     STRING = [VERSION, PRE].compact.join("-")
   end
 
+  # @!visibility private
   VERSION = Version::STRING
+
+  # @!visibility private
+  PROTOCOL = 1
 end

data/lib/falqon.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# typed: true
+
+require "forwardable"
+
+require "sorbet-runtime"
+require "zeitwerk"
+
+module Falqon
+  class << self
+    extend Forwardable
+    extend T::Sig
+
+    # Code loader instance
+    # @!visibility private
+    attr_reader :loader
+
+    # Global configuration
+    #
+    # @see Falqon::Configuration
+    sig { returns(Configuration) }
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    # @!visibility private
+    def root
+      @root ||= Pathname.new(File.expand_path(File.join("..", ".."), __FILE__))
+    end
+
+    # @!visibility private
+    def setup
+      @loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false)
+
+      # Register inflections
+      require root.join("config/inflections.rb")
+
+      # Collapse concerns directory
+      loader.collapse(root.join("lib/falqon/concerns"))
+
+      # Configure Rails generators (if applicable)
+      if const_defined?(:Rails)
+        loader.collapse(root.join("lib/generators"))
+      else
+        loader.ignore(root.join("lib/generators"))
+      end
+
+      loader.setup
+      loader.eager_load
+    end
+
+    # @!visibility private
+    def configure
+      yield configuration
+    end
+
+    def_delegator :configuration, :redis
+    def_delegator :configuration, :logger
+  end
+end
+
+Falqon.setup

data/lib/generators/falqon/install.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Falqon
+  # @!visibility private
+  class Install < Rails::Generators::Base
+    source_root File.expand_path("templates", __dir__)
+
+    def create_initializer_file
+      create_file "config/initializers/falqon.rb", <<~RUBY
+        Falqon.configure do |config|
+          # Configure global queue name prefix
+          # config.prefix = ENV.fetch("FALQON_PREFIX", "falqon")
+
+          # Retry strategy (none or linear)
+          # config.retry_strategy = :linear
+
+          # Maximum number of retries before a message is discarded (-1 for infinite retries)
+          # config.max_retries = 3
+
+          # Retry delay (in seconds) for linear retry strategy (defaults to 0)
+          # config.retry_delay = 60
+
+          # Configure the Redis client options
+          # config.redis_options = { url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0") }
+
+          # Or, configure the Redis client directly
+          # config.redis = ConnectionPool.new(size: 5, timeout: 5) { Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) }
+
+          # Configure logger
+          # config.logger = Logger.new(STDOUT)
+        end
+      RUBY
+    end
+  end
+end

metadata

@@ -1,33 +1,90 @@
 --- !ruby/object:Gem::Specification
 name: falqon
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Florian Dejonckheere
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-22 00:00:00.000000000 Z
+date: 2024-12-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: sorbet-runtime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: Simple, efficient messaging queue for Ruby
+        version: '2.6'
+description: Simple, efficient, and reliable messaging queue for Ruby
 email:
 - florian@floriandejonckheere.be
-executables: []
+executables:
+- falqon
 extensions: []
 extra_rdoc_files: []
 files:
@@ -35,15 +92,45 @@ files:
 - Gemfile
 - LICENSE.md
 - README.md
+- bin/falqon
 - config/inflections.rb
+- lib/falqon.rb
+- lib/falqon/cli.rb
+- lib/falqon/cli/base.rb
+- lib/falqon/cli/clear.rb
+- lib/falqon/cli/delete.rb
+- lib/falqon/cli/kill.rb
+- lib/falqon/cli/list.rb
+- lib/falqon/cli/refill.rb
+- lib/falqon/cli/revive.rb
+- lib/falqon/cli/schedule.rb
+- lib/falqon/cli/show.rb
+- lib/falqon/cli/stats.rb
+- lib/falqon/cli/status.rb
+- lib/falqon/cli/version.rb
+- lib/falqon/concerns/hooks.rb
+- lib/falqon/configuration.rb
+- lib/falqon/connection_pool_snooper.rb
+- lib/falqon/data.rb
+- lib/falqon/error.rb
+- lib/falqon/identifier.rb
+- lib/falqon/message.rb
+- lib/falqon/middlewares/logger.rb
+- lib/falqon/queue.rb
+- lib/falqon/strategies/linear.rb
+- lib/falqon/strategies/none.rb
+- lib/falqon/strategy.rb
+- lib/falqon/sub_queue.rb
+- lib/falqon/sub_set.rb
 - lib/falqon/version.rb
+- lib/generators/falqon/install.rb
 homepage: https://github.com/floriandejonckheere/falqon
 licenses:
 - LGPL-3.0
 metadata:
   source_code_uri: https://github.com/floriandejonckheere/falqon.git
   rubygems_mfa_required: 'true'
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -51,15 +138,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
-signing_key: 
+rubygems_version: 3.4.20
+signing_key:
 specification_version: 4
 summary: Simple messaging queue
 test_files: []