tina4ruby 3.13.40 → 3.13.41

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0bb742a8cdfa909ee27c95cce703f14f678f03f93b764aea62501b4afde78e91
-  data.tar.gz: 3e0da501f8911d1df438ecebdab6b2f8ef67be7b66d39927672c23f089b351c4
+  metadata.gz: 74de2870c01871240bb98d6ceeb66cd5bc5bf05c6e1c4e82d19a198d585d83e6
+  data.tar.gz: 9d548edf025f53d6547216f3a421d9215eb893cfda89131c342910ebc2c820bb
 SHA512:
-  metadata.gz: 1f6998cc599c7ebae85a3a0020b8146721ca653a84f7f65635226072781aac2ad41ccec3cf3a749063960e3f1bae47c3e3eca97fe58375d6c01c0a5df7359487
-  data.tar.gz: ce0653713c6f9e0659b5f26a6068690db7915ccbc8cdeb87ca54c8cfe609c2b700f6e90a5f6ca90b8f6c198d33edbcea70599bb050855259504fd831f16b90f6
+  metadata.gz: 3d2db7a1fc7dfcb95848dbc80c4f76664800bd6692a460667f5dfcbb59cb59fcba101737773c4eda4b4731275f735b4ef494164388869ded0849d29bb6de3804
+  data.tar.gz: 49e9c4742313a45ebb610cc5840c1b277915db9b1a5eb3a98dcb5b680a93d336b5b1d846698a9ec9121d2d722c905b8e0261b2532d050d0d7805b27a6c2c5228

data/lib/tina4/queue.rb

@@ -17,17 +17,31 @@ module Tina4
   #   # Or pass a backend instance directly (legacy)
   #   queue = Queue.new(topic: "tasks", backend: my_backend)
   class Queue
-    attr_reader :topic, :max_retries, :retry_backoff
+    attr_reader :topic, :max_retries, :retry_backoff, :visibility_timeout
 
-    def initialize(topic:, backend: nil, max_retries: 3, retry_backoff: 0)
+    def initialize(topic:, backend: nil, max_retries: 3, retry_backoff: 0, visibility_timeout: nil)
       @topic = topic
       @max_retries = max_retries
       # Seconds to wait before a failed job is re-attempted (lite backend).
       # Default 0 = retry on the very next pop/consume iteration.
       @retry_backoff = retry_backoff
+      # Reservation/visibility timeout (seconds). A popped job is reserved for
+      # this long; if the consumer dies before complete()/fail() the next pop()
+      # reclaims it (at-least-once delivery). Falls back to
+      # TINA4_QUEUE_VISIBILITY_TIMEOUT, else 300 (5 min). <= 0 disables reclaim.
+      # RabbitMQ/Kafka ignore it — the broker owns redelivery.
+      @visibility_timeout =
+        visibility_timeout.nil? ? self.class.default_visibility_timeout : visibility_timeout.to_f
       @backend = resolve_backend_arg(backend)
     end
 
+    # Reservation/visibility timeout in seconds, from env (default 300 = 5 min).
+    def self.default_visibility_timeout
+      Float(ENV.fetch("TINA4_QUEUE_VISIBILITY_TIMEOUT", "300"))
+    rescue ArgumentError, TypeError
+      300.0
+    end
+
     # Push a job onto the queue. Returns the Job.
     # priority: higher-priority messages are dequeued first (default 0).
     # delay_seconds: delay before the message becomes available (default 0).
@@ -207,6 +221,8 @@ module Tina4
       case status.to_s
       when "pending"
         @backend.size(@topic)
+      when "reserved"
+        @backend.respond_to?(:reserved_count) ? @backend.reserved_count(@topic) : 0
       when "failed", "dead"
         if @backend.respond_to?(:dead_letter_count)
           @backend.dead_letter_count(@topic)
@@ -267,20 +283,28 @@ module Tina4
     end
 
     # Resolve the default backend from env vars.
-    def self.resolve_backend(name = nil, max_retries: 3, retry_backoff: 0)
+    def self.resolve_backend(name = nil, max_retries: 3, retry_backoff: 0, visibility_timeout: nil)
       chosen = name || ENV.fetch("TINA4_QUEUE_BACKEND", "file").downcase.strip
+      vt = visibility_timeout.nil? ? default_visibility_timeout : visibility_timeout
 
       case chosen.to_s
       when "lite", "file", "default"
-        Tina4::QueueBackends::LiteBackend.new(max_retries: max_retries, retry_backoff: retry_backoff)
+        Tina4::QueueBackends::LiteBackend.new(
+          max_retries: max_retries, retry_backoff: retry_backoff, visibility_timeout: vt
+        )
       when "rabbitmq"
+        # Broker manages visibility/redelivery (unacked messages requeue on
+        # channel close) — the framework timeout is accepted but not used.
         config = resolve_rabbitmq_config
         Tina4::QueueBackends::RabbitmqBackend.new(config)
       when "kafka"
+        # Consumer-group offsets manage redelivery — framework timeout N/A.
         config = resolve_kafka_config
         Tina4::QueueBackends::KafkaBackend.new(config)
       when "mongodb", "mongo"
         config = resolve_mongo_config
+        config[:visibility_timeout] = vt
+        config[:max_retries] = max_retries
         Tina4::QueueBackends::MongoBackend.new(config)
       else
         raise ArgumentError, "Unknown queue backend: #{chosen.inspect}. Use 'lite', 'rabbitmq', 'kafka', or 'mongodb'."
@@ -302,10 +326,12 @@ module Tina4
       if backend && !backend.is_a?(Symbol) && !backend.is_a?(String)
         backend.max_retries = @max_retries if backend.respond_to?(:max_retries=)
         backend.retry_backoff = @retry_backoff if backend.respond_to?(:retry_backoff=)
+        backend.visibility_timeout = @visibility_timeout if backend.respond_to?(:visibility_timeout=)
         return backend
       end
       # If a symbol or string name is passed, resolve it
-      Queue.resolve_backend(backend, max_retries: @max_retries, retry_backoff: @retry_backoff)
+      Queue.resolve_backend(backend, max_retries: @max_retries, retry_backoff: @retry_backoff,
+                                     visibility_timeout: @visibility_timeout)
     end
 
     def self.resolve_rabbitmq_config

data/lib/tina4/queue_backends/lite_backend.rb

@@ -28,11 +28,23 @@ module Tina4
         # Seconds to delay a job's next attempt when fail() re-enqueues it.
         # 0 (default) = retry on the very next pop/consume iteration.
         @retry_backoff = options[:retry_backoff] || 0
+        # Reservation/visibility timeout (seconds). When a job is popped it is
+        # held in <topic>/reserved/ with available_at = now + visibility_timeout.
+        # If the consumer dies before complete()/fail() (crash, OOM, k8s
+        # eviction) the next pop() reclaims it once the window expires —
+        # incrementing attempts and re-enqueuing, or dead-lettering past
+        # max_retries. <= 0 disables the reclaim (a reservation then lasts until
+        # the consumer acks — the old at-most-once behaviour).
+        @visibility_timeout = options[:visibility_timeout] || 300.0
         FileUtils.mkdir_p(@dir)
         FileUtils.mkdir_p(@dead_letter_dir)
         @mutex = Mutex.new
       end
 
+      # Retry/visibility policy is settable so a Queue can propagate its own
+      # configuration onto a backend instance passed directly (legacy path).
+      attr_accessor :visibility_timeout
+
       def enqueue(message)
         @mutex.synchronize do
           topic_dir = topic_path(message.topic)
@@ -44,19 +56,35 @@ module Tina4
 
       def dequeue(topic)
         @mutex.synchronize do
+          # First return any reservations whose consumer died mid-flight.
+          reclaim_expired(topic)
           candidate = available_candidates(topic).first
           return nil unless candidate
 
-          File.delete(candidate[:file])
+          # Write the reservation BEFORE claiming the pending file, so a crash
+          # between claim and reserve can never strand the job. Only the worker
+          # that wins the delete owns — and returns — it.
+          write_reserved(candidate[:data], topic)
+          begin
+            File.delete(candidate[:file])
+          rescue Errno::ENOENT
+            return nil # already consumed by another worker
+          end
           job_from_data(candidate[:data], topic)
         end
       end
 
       def dequeue_batch(topic, count)
         @mutex.synchronize do
+          reclaim_expired(topic)
           chosen = available_candidates(topic).first(count)
-          chosen.map do |c|
-            File.delete(c[:file])
+          chosen.filter_map do |c|
+            write_reserved(c[:data], topic)
+            begin
+              File.delete(c[:file])
+            rescue Errno::ENOENT
+              next
+            end
             job_from_data(c[:data], topic)
           end
         end
@@ -74,6 +102,9 @@ module Tina4
             data = JSON.parse(File.read(f))
             next unless data["id"].to_s == target
 
+            # Reserve (so a dead consumer's job is reclaimable) then claim the
+            # pending file — mirrors dequeue.
+            write_reserved(data, topic)
             File.delete(f)
             return job_from_data(data, topic)
           rescue JSON::ParserError
@@ -88,8 +119,10 @@ module Tina4
       end
 
       def complete(message)
-        # Job file was already deleted on dequeue. complete() is terminal:
-        # the job is done and gone.
+        # The pending file was claimed on dequeue and a reservation record
+        # written; complete() is terminal, so drop the reservation. The job is
+        # done and gone.
+        clear_reservation(message.topic, message.id)
       end
 
       def requeue(message)
@@ -101,6 +134,8 @@ module Tina4
       # the configured retry_backoff). Once attempts >= max_retries it is moved
       # to the dead-letter store.
       def fail(job, error = "")
+        # Clear the reservation — the consumer acknowledged (with a failure).
+        clear_reservation(job.topic, job.id)
         job.attempts += 1
         job.error = error
         if job.attempts < @max_retries
@@ -114,6 +149,7 @@ module Tina4
       # re-enqueues regardless of the retry limit — a manual override, distinct
       # from the automatic fail() path. Increments attempts, clears the error.
       def retry(job, delay_seconds: 0)
+        clear_reservation(job.topic, job.id)
         job.attempts += 1
         requeue_job(job, delay_seconds: delay_seconds, error: nil)
       end
@@ -131,6 +167,13 @@ module Tina4
         Dir.glob(File.join(dir, "*.json")).length
       end
 
+      # Count currently-reserved (in-flight) jobs for a topic.
+      def reserved_count(topic)
+        dir = reserved_path(topic)
+        return 0 unless Dir.exist?(dir)
+        Dir.glob(File.join(dir, "*.json")).length
+      end
+
       # Count dead-letter / failed messages for a topic.
       def dead_letter_count(topic)
         return 0 unless Dir.exist?(@dead_letter_dir)
@@ -241,14 +284,18 @@ module Tina4
         count
       end
 
-      # Remove all pending jobs from a topic. Returns count removed.
+      # Remove all pending jobs from a topic (and any held reservations).
+      # Returns count removed.
       def clear(topic)
-        dir = topic_path(topic)
-        return 0 unless Dir.exist?(dir)
         count = 0
-        Dir.glob(File.join(dir, "*.json")).each do |file|
-          File.delete(file)
-          count += 1
+        [topic_path(topic), reserved_path(topic)].each do |dir|
+          next unless Dir.exist?(dir)
+          Dir.glob(File.join(dir, "*.json")).each do |file|
+            File.delete(file)
+            count += 1
+          rescue Errno::ENOENT
+            next
+          end
         end
         count
       end
@@ -366,6 +413,13 @@ module Tina4
       # within a priority tier it sorts behind jobs not yet attempted) and the
       # current attempts/error carried over.
       def requeue_job(job, delay_seconds: 0, error: nil)
+        @mutex.synchronize { write_pending(job, delay_seconds: delay_seconds, error: error) }
+      end
+
+      # Mutex-free pending write. Callers already holding @mutex (e.g.
+      # reclaim_expired, invoked from within dequeue's locked block) use this
+      # directly — Ruby's Mutex is non-reentrant, so re-locking would deadlock.
+      def write_pending(job, delay_seconds: 0, error: nil)
         available_at = delay_seconds > 0 ? (Time.now + delay_seconds).iso8601(6) : nil
         data = {
           id: job.id,
@@ -378,16 +432,19 @@ module Tina4
           created_at: Time.now.iso8601(6)
         }
         data[:available_at] = available_at if available_at
-        @mutex.synchronize do
-          topic_dir = topic_path(job.topic)
-          FileUtils.mkdir_p(topic_dir)
-          File.write(File.join(topic_dir, "#{job.id}.json"), JSON.generate(data))
-        end
+        topic_dir = topic_path(job.topic)
+        FileUtils.mkdir_p(topic_dir)
+        File.write(File.join(topic_dir, "#{job.id}.json"), JSON.generate(data))
       end
 
       # Move a failed job to the dead-letter directory. Terminal until a manual
       # retry_failed/retry revives it.
       def move_to_dead_letter(job, error = "")
+        @mutex.synchronize { write_dead_letter(job, error) }
+      end
+
+      # Mutex-free dead-letter write (see write_pending for the re-entrancy note).
+      def write_dead_letter(job, error = "")
         data = {
           id: job.id,
           topic: job.topic,
@@ -398,16 +455,97 @@ module Tina4
           error: error,
           failed_at: Time.now.iso8601(6)
         }
-        @mutex.synchronize do
-          FileUtils.mkdir_p(@dead_letter_dir)
-          File.write(File.join(@dead_letter_dir, "#{job.id}.json"), JSON.generate(data))
-        end
+        FileUtils.mkdir_p(@dead_letter_dir)
+        File.write(File.join(@dead_letter_dir, "#{job.id}.json"), JSON.generate(data))
       end
 
       def topic_path(topic)
         safe_topic = topic.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
         File.join(@dir, safe_topic)
       end
+
+      # Directory holding a topic's reservation records (in-flight jobs).
+      def reserved_path(topic)
+        File.join(topic_path(topic), "reserved")
+      end
+
+      # Persist a reservation record so a dead consumer's job is reclaimable.
+      # Stores reserved_at + available_at = now + visibility_timeout. The next
+      # dequeue reclaims this job once available_at has passed (see
+      # reclaim_expired). complete()/fail()/retry() delete the record.
+      def write_reserved(data, topic)
+        now = Time.now
+        vt = @visibility_timeout || 0
+        record = {
+          id: data["id"],
+          topic: data["topic"] || topic.to_s,
+          payload: data["payload"],
+          status: "reserved",
+          priority: data["priority"] || 0,
+          attempts: data["attempts"] || 0,
+          error: data["error"],
+          reserved_at: now.iso8601(6),
+          available_at: (vt > 0 ? (now + vt) : now).iso8601(6),
+          created_at: data["created_at"] || now.iso8601(6)
+        }
+        dir = reserved_path(topic)
+        FileUtils.mkdir_p(dir)
+        File.write(File.join(dir, "#{record[:id]}.json"), JSON.generate(record))
+      end
+
+      # Delete a job's reservation record (best-effort).
+      def clear_reservation(topic, id)
+        File.delete(File.join(reserved_path(topic), "#{id}.json"))
+      rescue Errno::ENOENT
+        nil
+      end
+
+      # Return expired reservations to the queue (at-least-once delivery).
+      #
+      # A reserved job whose available_at <= now means its consumer never
+      # acknowledged in time (crash / OOM / pod eviction). Increment attempts and
+      # either re-enqueue it (so the next dequeue picks it up) or dead-letter it
+      # once it has hit max_retries. Disabled when visibility_timeout <= 0.
+      def reclaim_expired(topic)
+        return if @visibility_timeout.nil? || @visibility_timeout <= 0
+
+        dir = reserved_path(topic)
+        return unless Dir.exist?(dir)
+
+        now = Time.now
+        Dir.glob(File.join(dir, "*.json")).each do |file|
+          data = JSON.parse(File.read(file))
+          available_at = data["available_at"] ? Time.parse(data["available_at"]) : now
+          next if available_at > now # reservation still valid
+
+          # Atomically claim the expired reservation by deleting its file.
+          begin
+            File.delete(file)
+          rescue Errno::ENOENT
+            next # another worker reclaimed it first
+          end
+
+          attempts = (data["attempts"] || 0) + 1
+          error = "reservation timed out - consumer did not acknowledge within the visibility timeout"
+          job = Tina4::Job.new(
+            topic: data["topic"] || topic.to_s,
+            payload: data["payload"],
+            id: data["id"],
+            priority: data["priority"] || 0,
+            attempts: attempts,
+            error: error
+          )
+          # Mutex-free writers: reclaim_expired runs inside dequeue's locked
+          # block, and Ruby's Mutex is non-reentrant.
+          if attempts >= @max_retries
+            write_dead_letter(job, error)
+          else
+            write_pending(job, delay_seconds: 0, error: error)
+          end
+        rescue JSON::ParserError
+          next
+        end
+      end
     end
   end
 end

data/lib/tina4/queue_backends/mongo_backend.rb

@@ -3,8 +3,13 @@
 module Tina4
   module QueueBackends
     class MongoBackend
+      # Reservation/visibility + retry policy (settable so a Queue can propagate
+      # its own onto a backend instance passed directly — legacy path).
+      attr_accessor :visibility_timeout, :max_retries
+
       def initialize(options = {})
         require "mongo"
+        @max_retries = options[:max_retries] || 3
 
         uri = options[:uri] || ENV["TINA4_MONGO_URI"]
         host = options[:host] || ENV.fetch("TINA4_MONGO_HOST", "localhost")
@@ -13,6 +18,11 @@ module Tina4
         password = options[:password] || ENV["TINA4_MONGO_PASSWORD"]
         db_name = options[:db] || ENV.fetch("TINA4_MONGO_DB", "tina4")
         @collection_name = options[:collection] || ENV.fetch("TINA4_MONGO_COLLECTION", "tina4_queue")
+        # Reservation/visibility timeout (seconds): a dequeued message is held
+        # reserved (status "processing") with available_at = now + timeout;
+        # reclaim_expired returns it once that passes (consumer died mid-flight).
+        # <= 0 disables reclaim.
+        @visibility_timeout = resolve_visibility_timeout(options[:visibility_timeout])
 
         if uri
           @client = Mongo::Client.new(uri)
@@ -29,6 +39,14 @@ module Tina4
         raise "MongoDB backend requires the 'mongo' gem. Install with: gem install mongo"
       end
 
+      def resolve_visibility_timeout(option)
+        return option.to_f unless option.nil?
+
+        Float(ENV.fetch("TINA4_QUEUE_VISIBILITY_TIMEOUT", "300"))
+      rescue ArgumentError, TypeError
+        300.0
+      end
+
       def enqueue(message)
         collection.insert_one(
           _id: message.id,
@@ -41,9 +59,21 @@ module Tina4
       end
 
       def dequeue(topic)
+        # Reclaim any reservations whose consumer died before acking, then take
+        # the next available message (at-least-once delivery).
+        reclaim_expired(topic, @max_retries)
+        now = Time.now.utc
+        # The claim advances available_at to now + visibility_timeout and records
+        # reserved_at so reclaim_expired can return the job if the consumer dies
+        # before acknowledge/complete. This is the fix for the "reserved forever"
+        # bug — previously available_at was left unchanged.
         doc = collection.find_one_and_update(
           { topic: topic, status: "pending" },
-          { "$set" => { status: "processing" } },
+          { "$set" => {
+            status: "processing",
+            reserved_at: now,
+            available_at: now + (@visibility_timeout || 0)
+          } },
           sort: { created_at: 1 },
           return_document: :after
         )
@@ -52,10 +82,54 @@ module Tina4
         Tina4::Job.new(
           topic: doc["topic"],
           payload: doc["payload"],
-          id: doc["_id"]
+          id: doc["_id"],
+          priority: doc["priority"] || 0,
+          attempts: doc["attempts"] || 0
         )
       end
 
+      # Return reservations whose visibility window expired (at-least-once).
+      #
+      # A message left "processing" with available_at <= now had a consumer die
+      # before acknowledging. Each is atomically flipped back to "pending" with
+      # attempts incremented (so the next dequeue re-delivers it); once attempts
+      # >= max_retries it is dead-lettered instead. Returns the number reclaimed.
+      # Disabled when visibility_timeout <= 0.
+      def reclaim_expired(topic, max_retries)
+        return 0 if @visibility_timeout.nil? || @visibility_timeout <= 0
+
+        reclaimed = 0
+        loop do
+          now = Time.now.utc
+          doc = collection.find_one_and_update(
+            { topic: topic, status: "processing", available_at: { "$lte" => now } },
+            { "$set" => { status: "pending", available_at: now, reserved_at: nil },
+              "$inc" => { attempts: 1 } },
+            sort: { available_at: 1 },
+            return_document: :after
+          )
+          break unless doc
+
+          reclaimed += 1
+          next if (doc["attempts"] || 0) < max_retries
+
+          # Out of retries — move it to the dead-letter queue and remove the
+          # original so it is not re-delivered.
+          collection.insert_one(
+            _id: "#{doc["_id"]}.dead_letter",
+            topic: "#{topic}.dead_letter",
+            payload: doc["payload"],
+            status: "dead",
+            priority: doc["priority"] || 0,
+            attempts: doc["attempts"] || 0,
+            error: "reservation timed out - consumer did not acknowledge within the visibility timeout",
+            created_at: Time.now.utc
+          )
+          collection.delete_one(_id: doc["_id"], topic: topic)
+        end
+        reclaimed
+      end
+
       def acknowledge(message)
         collection.delete_one(_id: message.id)
       end

data/lib/tina4/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tina4
-  VERSION = "3.13.40"
+  VERSION = "3.13.41"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tina4ruby
 version: !ruby/object:Gem::Version
-  version: 3.13.40
+  version: 3.13.41
 platform: ruby
 authors:
 - Tina4 Team