tina4ruby 3.0.0 → 3.9.2

data/lib/tina4/queue.rb

@@ -4,27 +4,42 @@ require "securerandom"
 
 module Tina4
   class QueueMessage
-    attr_reader :id, :topic, :payload, :created_at, :attempts
+    attr_reader :id, :topic, :payload, :created_at, :attempts, :priority, :available_at
     attr_accessor :status
 
-    def initialize(topic:, payload:, id: nil)
+    def initialize(topic:, payload:, id: nil, priority: 0, available_at: nil, attempts: 0)
       @id = id || SecureRandom.uuid
       @topic = topic
       @payload = payload
       @created_at = Time.now
-      @attempts = 0
+      @attempts = attempts
+      @priority = priority
+      @available_at = available_at
       @status = :pending
     end
 
+    # Re-queue this message with incremented attempts.
+    # Delegates to the queue's backend via the queue reference.
+    def retry(queue:, delay_seconds: 0)
+      @attempts += 1
+      @status = :pending
+      @available_at = delay_seconds > 0 ? Time.now + delay_seconds : nil
+      queue.backend.enqueue(self)
+      self
+    end
+
     def to_hash
-      {
+      h = {
         id: @id,
         topic: @topic,
         payload: @payload,
         created_at: @created_at.iso8601,
         attempts: @attempts,
-        status: @status
+        status: @status,
+        priority: @priority
       }
+      h[:available_at] = @available_at.iso8601 if @available_at
+      h
     end
 
     def to_json(*_args)
@@ -34,34 +49,61 @@ module Tina4
     def increment_attempts!
       @attempts += 1
     end
-  end
 
-  class Producer
-    def initialize(backend: nil)
-      @backend = backend || Tina4::QueueBackends::LiteBackend.new
+    # Mark this job as completed.
+    def complete
+      @status = :completed
     end
 
-    def publish(topic, payload)
-      message = QueueMessage.new(topic: topic, payload: payload)
-      @backend.enqueue(message)
-      Tina4::Log.debug("Message published to #{topic}: #{message.id}")
-      message
+    # Mark this job as failed with a reason.
+    def fail(reason = "")
+      @status = :failed
+      @error = reason
+      @attempts += 1
     end
 
-    def publish_batch(topic, payloads)
-      payloads.map { |p| publish(topic, p) }
+    # Reject this job with a reason. Alias for fail().
+    def reject(reason = "")
+      fail(reason)
     end
+
+    attr_reader :error
   end
 
-  # Queue — convenience wrapper for queue management operations.
-  # Provides dead letter inspection, purging, and retry capabilities.
+  # Queue — unified wrapper for queue management operations.
+  # Auto-detects backend from TINA4_QUEUE_BACKEND env var.
+  #
+  # Usage:
+  #   # Auto-detect from env (default: lite/file backend)
+  #   queue = Queue.new(topic: "tasks")
+  #
+  #   # Explicit backend
+  #   queue = Queue.new(topic: "tasks", backend: :rabbitmq)
+  #
+  #   # Or pass a backend instance directly (legacy)
+  #   queue = Queue.new(topic: "tasks", backend: my_backend)
   class Queue
     attr_reader :topic, :max_retries
 
     def initialize(topic:, backend: nil, max_retries: 3)
       @topic = topic
-      @backend = backend || Tina4::QueueBackends::LiteBackend.new
       @max_retries = max_retries
+      @backend = resolve_backend_arg(backend)
+    end
+
+    # Push a job onto the queue. Returns the QueueMessage.
+    # priority: higher-priority messages are dequeued first (default 0).
+    # delay_seconds: delay before the message becomes available (default 0).
+    def push(payload, priority: 0, delay_seconds: 0)
+      available_at = delay_seconds > 0 ? Time.now + delay_seconds : nil
+      message = QueueMessage.new(topic: @topic, payload: payload, priority: priority, available_at: available_at)
+      @backend.enqueue(message)
+      message
+    end
+
+    # Pop the next available job. Returns QueueMessage or nil.
+    def pop
+      @backend.dequeue(@topic)
     end
 
     # Get dead letter jobs — messages that exceeded max retries.
@@ -83,71 +125,187 @@ module Tina4
       @backend.retry_failed(@topic, max_retries: @max_retries)
     end
 
-    # Get the number of pending messages.
-    def size
-      @backend.size(@topic)
+    # Produce a message onto a topic. Convenience wrapper around push().
+    def produce(topic, payload)
+      message = QueueMessage.new(topic: topic, payload: payload)
+      @backend.enqueue(message)
+      message
     end
-  end
 
-  class Consumer
-    def initialize(topic:, backend: nil, max_retries: 3)
-      @topic = topic
-      @backend = backend || Tina4::QueueBackends::LiteBackend.new
-      @max_retries = max_retries
-      @handlers = []
-      @running = false
-    end
+    # Consume jobs from a topic using an Enumerator (yield pattern).
+    #
+    # Usage:
+    #   queue.consume("emails") do |job|
+    #     process(job)
+    #   end
+    #
+    #   # Consume a specific job by ID:
+    #   queue.consume("emails", id: "abc-123") do |job|
+    #     process(job)
+    #   end
+    #
+    #   # Or as an enumerator:
+    #   queue.consume("emails").each { |job| process(job) }
+    #
+    def consume(topic = nil, id: nil, &block)
+      topic ||= @topic
 
-    def on_message(&block)
-      @handlers << block
+      if block_given?
+        if id
+          job = pop_by_id(topic, id)
+          yield job if job
+        else
+          while (job = @backend.dequeue(topic))
+            yield job
+          end
+        end
+      else
+        # Return an Enumerator when no block given
+        Enumerator.new do |yielder|
+          if id
+            job = pop_by_id(topic, id)
+            yielder << job if job
+          else
+            while (job = @backend.dequeue(topic))
+              yielder << job
+            end
+          end
+        end
+      end
     end
 
-    def start(poll_interval: 1)
-      @running = true
-      Tina4::Log.info("Consumer started for topic: #{@topic}")
+    # Pop a specific job by ID from the queue.
+    def pop_by_id(topic, id)
+      return nil unless @backend.respond_to?(:find_by_id)
+      @backend.find_by_id(topic, id)
+    end
 
-      while @running
-        message = @backend.dequeue(@topic)
-        if message
-          process_message(message)
+    # Get the number of messages by status.
+    # status: "pending" (default) counts pending messages in the topic queue.
+    # status: "failed" or "dead" counts messages in the dead_letter directory.
+    def size(status: "pending")
+      case status.to_s
+      when "pending"
+        @backend.size(@topic)
+      when "failed", "dead"
+        if @backend.respond_to?(:dead_letter_count)
+          @backend.dead_letter_count(@topic)
         else
-          sleep(poll_interval)
+          0
         end
+      else
+        @backend.size(@topic)
       end
     end
 
-    def stop
-      @running = false
-      Tina4::Log.info("Consumer stopped for topic: #{@topic}")
+    # Get the underlying backend instance.
+    def backend
+      @backend
     end
 
-    def process_one
-      message = @backend.dequeue(@topic)
-      process_message(message) if message
+    # Resolve the default backend from env vars.
+    def self.resolve_backend(name = nil)
+      chosen = name || ENV.fetch("TINA4_QUEUE_BACKEND", "file").downcase.strip
+
+      case chosen.to_s
+      when "lite", "file", "default"
+        Tina4::QueueBackends::LiteBackend.new
+      when "rabbitmq"
+        config = resolve_rabbitmq_config
+        Tina4::QueueBackends::RabbitmqBackend.new(config)
+      when "kafka"
+        config = resolve_kafka_config
+        Tina4::QueueBackends::KafkaBackend.new(config)
+      when "mongodb", "mongo"
+        config = resolve_mongo_config
+        Tina4::QueueBackends::MongoBackend.new(config)
+      else
+        raise ArgumentError, "Unknown queue backend: #{chosen.inspect}. Use 'lite', 'rabbitmq', 'kafka', or 'mongodb'."
+      end
     end
 
     private
 
-    def process_message(message)
-      message.increment_attempts!
-      message.status = :processing
+    def resolve_backend_arg(backend)
+      # If a backend instance is passed directly (legacy), use it
+      return backend if backend && !backend.is_a?(Symbol) && !backend.is_a?(String)
+      # If a symbol or string name is passed, resolve it
+      Queue.resolve_backend(backend)
+    end
+
+    def self.resolve_rabbitmq_config
+      config = {}
+      url = ENV["TINA4_QUEUE_URL"]
+      if url
+        config = parse_amqp_url(url)
+      end
+      config[:host] ||= ENV.fetch("TINA4_RABBITMQ_HOST", "localhost")
+      config[:port] ||= (ENV["TINA4_RABBITMQ_PORT"] || 5672).to_i
+      config[:username] ||= ENV.fetch("TINA4_RABBITMQ_USERNAME", "guest")
+      config[:password] ||= ENV.fetch("TINA4_RABBITMQ_PASSWORD", "guest")
+      config[:vhost] ||= ENV.fetch("TINA4_RABBITMQ_VHOST", "/")
+      config
+    end
 
-      @handlers.each do |handler|
-        handler.call(message)
+    def self.resolve_kafka_config
+      config = {}
+      url = ENV["TINA4_QUEUE_URL"]
+      if url
+        config[:brokers] = url.sub("kafka://", "")
       end
+      brokers = ENV["TINA4_KAFKA_BROKERS"]
+      config[:brokers] = brokers if brokers
+      config[:brokers] ||= "localhost:9092"
+      config[:group_id] = ENV.fetch("TINA4_KAFKA_GROUP_ID", "tina4_consumer_group")
+      config
+    end
+
+    def self.resolve_mongo_config
+      config = {}
+      uri = ENV["TINA4_MONGO_URI"]
+      config[:uri] = uri if uri
+      config[:host] = ENV.fetch("TINA4_MONGO_HOST", "localhost") unless uri
+      config[:port] = (ENV["TINA4_MONGO_PORT"] || 27017).to_i unless uri
+      username = ENV["TINA4_MONGO_USERNAME"]
+      password = ENV["TINA4_MONGO_PASSWORD"]
+      config[:username] = username if username
+      config[:password] = password if password
+      config[:db] = ENV.fetch("TINA4_MONGO_DB", "tina4")
+      config[:collection] = ENV.fetch("TINA4_MONGO_COLLECTION", "tina4_queue")
+      config
+    end
+
+    def self.parse_amqp_url(url)
+      config = {}
+      url = url.sub("amqp://", "").sub("amqps://", "")
 
-      message.status = :completed
-      @backend.acknowledge(message)
-    rescue => e
-      Tina4::Log.error("Queue message failed: #{message.id} - #{e.message}")
-      message.status = :failed
+      if url.include?("@")
+        creds, rest = url.split("@", 2)
+        if creds.include?(":")
+          config[:username], config[:password] = creds.split(":", 2)
+        else
+          config[:username] = creds
+        end
+      else
+        rest = url
+      end
 
-      if message.attempts < @max_retries
-        message.status = :pending
-        @backend.requeue(message)
+      if rest.include?("/")
+        hostport, vhost = rest.split("/", 2)
+        config[:vhost] = vhost.start_with?("/") ? vhost : "/#{vhost}" if vhost && !vhost.empty?
       else
-        @backend.dead_letter(message)
+        hostport = rest
       end
+
+      if hostport.include?(":")
+        host, port = hostport.split(":", 2)
+        config[:host] = host
+        config[:port] = port.to_i
+      elsif hostport && !hostport.empty?
+        config[:host] = hostport
+      end
+
+      config
     end
   end
 end

data/lib/tina4/queue_backends/lite_backend.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require "json"
 require "fileutils"
+require "time"
 
 module Tina4
   module QueueBackends
@@ -27,17 +28,37 @@ module Tina4
           dir = topic_path(topic)
           return nil unless Dir.exist?(dir)
 
-          files = Dir.glob(File.join(dir, "*.json")).sort_by { |f| File.mtime(f) }
-          return nil if files.empty?
+          now = Time.now
+          candidates = []
 
-          file = files.first
-          data = JSON.parse(File.read(file))
-          File.delete(file)
+          Dir.glob(File.join(dir, "*.json")).each do |f|
+            data = JSON.parse(File.read(f))
+            # Skip messages that are not yet available (delayed)
+            if data["available_at"]
+              available_at = Time.parse(data["available_at"])
+              next if available_at > now
+            end
+            candidates << { file: f, data: data, priority: data["priority"] || 0, mtime: File.mtime(f) }
+          rescue JSON::ParserError
+            next
+          end
+
+          return nil if candidates.empty?
+
+          # Sort by priority descending (higher first), then by mtime ascending (oldest first)
+          candidates.sort_by! { |c| [-c[:priority], c[:mtime]] }
+
+          chosen = candidates.first
+          File.delete(chosen[:file])
+          data = chosen[:data]
 
           Tina4::QueueMessage.new(
-            topic: data["topic"],
+            topic: data["topic"] || topic.to_s,
             payload: data["payload"],
-            id: data["id"]
+            id: data["id"],
+            priority: data["priority"] || 0,
+            available_at: data["available_at"] ? Time.parse(data["available_at"]) : nil,
+            attempts: data["attempts"] || 0
           )
         end
       end
@@ -61,6 +82,20 @@ module Tina4
         Dir.glob(File.join(dir, "*.json")).length
       end
 
+      # Count dead-letter / failed messages for a topic.
+      def dead_letter_count(topic)
+        return 0 unless Dir.exist?(@dead_letter_dir)
+
+        count = 0
+        Dir.glob(File.join(@dead_letter_dir, "*.json")).each do |file|
+          data = JSON.parse(File.read(file))
+          count += 1 if data["topic"] == topic.to_s
+        rescue JSON::ParserError
+          next
+        end
+        count
+      end
+
       def topics
         return [] unless Dir.exist?(@dir)
         Dir.children(@dir)

data/lib/tina4/queue_backends/mongo_backend.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Tina4
+  module QueueBackends
+    class MongoBackend
+      def initialize(options = {})
+        require "mongo"
+
+        uri = options[:uri] || ENV["TINA4_MONGO_URI"]
+        host = options[:host] || ENV.fetch("TINA4_MONGO_HOST", "localhost")
+        port = (options[:port] || ENV.fetch("TINA4_MONGO_PORT", 27017)).to_i
+        username = options[:username] || ENV["TINA4_MONGO_USERNAME"]
+        password = options[:password] || ENV["TINA4_MONGO_PASSWORD"]
+        db_name = options[:db] || ENV.fetch("TINA4_MONGO_DB", "tina4")
+        @collection_name = options[:collection] || ENV.fetch("TINA4_MONGO_COLLECTION", "tina4_queue")
+
+        if uri
+          @client = Mongo::Client.new(uri)
+        else
+          conn_options = { database: db_name }
+          conn_options[:user] = username if username
+          conn_options[:password] = password if password
+          @client = Mongo::Client.new(["#{host}:#{port}"], conn_options)
+        end
+
+        @db = @client.database
+        create_indexes
+      rescue LoadError
+        raise "MongoDB backend requires the 'mongo' gem. Install with: gem install mongo"
+      end
+
+      def enqueue(message)
+        collection.insert_one(
+          _id: message.id,
+          topic: message.topic,
+          payload: message.payload,
+          created_at: message.created_at.utc,
+          attempts: message.attempts,
+          status: "pending"
+        )
+      end
+
+      def dequeue(topic)
+        doc = collection.find_one_and_update(
+          { topic: topic, status: "pending" },
+          { "$set" => { status: "processing" } },
+          sort: { created_at: 1 },
+          return_document: :after
+        )
+        return nil unless doc
+
+        Tina4::QueueMessage.new(
+          topic: doc["topic"],
+          payload: doc["payload"],
+          id: doc["_id"]
+        )
+      end
+
+      def acknowledge(message)
+        collection.delete_one(_id: message.id)
+      end
+
+      def requeue(message)
+        collection.find_one_and_update(
+          { _id: message.id },
+          { "$set" => { status: "pending" }, "$inc" => { attempts: 1 } },
+          upsert: true
+        )
+      end
+
+      def dead_letter(message)
+        collection.find_one_and_update(
+          { _id: message.id },
+          { "$set" => { status: "dead", topic: "#{message.topic}.dead_letter" } },
+          upsert: true
+        )
+      end
+
+      def size(topic)
+        collection.count_documents(topic: topic, status: "pending")
+      end
+
+      def dead_letters(topic, max_retries: 3)
+        collection.find(topic: "#{topic}.dead_letter", status: "dead").map do |doc|
+          Tina4::QueueMessage.new(
+            topic: doc["topic"],
+            payload: doc["payload"],
+            id: doc["_id"]
+          )
+        end
+      end
+
+      def purge(topic, status)
+        result = collection.delete_many(topic: topic, status: status.to_s)
+        result.deleted_count
+      end
+
+      def retry_failed(topic, max_retries: 3)
+        result = collection.update_many(
+          { topic: topic, status: "failed", attempts: { "$lt" => max_retries } },
+          { "$set" => { status: "pending" } }
+        )
+        result.modified_count
+      end
+
+      def close
+        @client&.close
+      end
+
+      private
+
+      def collection
+        @db[@collection_name]
+      end
+
+      def create_indexes
+        collection.indexes.create_many([
+          { key: { topic: 1, status: 1, created_at: 1 } },
+          { key: { topic: 1, status: 1, attempts: 1 } }
+        ])
+      rescue Mongo::Error => e
+        Tina4::Log.warning("MongoDB index creation failed: #{e.message}")
+      end
+    end
+  end
+end