tina4ruby 3.13.94 → 3.13.96

data/lib/tina4/queue_backends/lite_backend.rb

@@ -7,7 +7,9 @@ module Tina4
   module QueueBackends
     # File-based queue backend — JSON files on disk. Zero dependencies.
     #
-    # Each job is stored as a separate .json file under <dir>/<topic>/.
+    # Each job is stored as a separate .queue-data file under <dir>/<topic>/,
+    # where <dir> is Tina4::Queue.base_path (TINA4_QUEUE_PATH, else data/queue)
+    # — the canonical cross-framework layout Python, PHP and Node all use.
     # Dead-lettered jobs (those that exhausted their retries) live under the
     # shared <dir>/dead_letter/ directory, tagged with their topic.
     #
@@ -15,13 +17,27 @@ module Tina4
     # stored created_at (ISO-8601, so lexicographic == chronological). The
     # file name is NOT the ordering key — the stored priority/created_at are.
     class LiteBackend
+      # The store's layout — extension, topic segment, dead-letter directory —
+      # is defined once on Tina4::Queue and shared with the dev-admin panel
+      # that reads the same files.
+      JOB_EXTENSION = Tina4::Queue::JOB_EXTENSION
+
+      # The pre-alignment Ruby store: Dir.pwd/.queue/<topic>/<id>.json. Read
+      # once per backend so an upgrading app's jobs are rescued rather than
+      # stranded (see migrate_legacy_store).
+      LEGACY_DIRNAME = ".queue"
+      LEGACY_JOB_EXTENSION = ".json"
+
       # Retry policy — settable so a Queue can propagate its own max_retries /
       # retry_backoff onto a backend instance passed directly (legacy path).
       attr_accessor :max_retries, :retry_backoff
 
       def initialize(options = {})
-        @dir = options[:dir] || File.join(Dir.pwd, ".queue")
-        @dead_letter_dir = File.join(@dir, "dead_letter")
+        # An explicit dir: is the caller's own store — take it as given. Only a
+        # store we RESOLVED is one the legacy rescue may touch.
+        resolved_store = options[:dir].nil?
+        @dir = options[:dir] || Tina4::Queue.base_path
+        @dead_letter_dir = File.join(@dir, Tina4::Queue::DEAD_LETTER_DIRNAME)
         # Retry policy. Mirrors the Python lite backend: a failed job is
         # re-enqueued while attempts < max_retries, then dead-lettered.
         @max_retries = options[:max_retries] || 3
@@ -39,6 +55,7 @@ module Tina4
         FileUtils.mkdir_p(@dir)
         FileUtils.mkdir_p(@dead_letter_dir)
         @mutex = Mutex.new
+        migrate_legacy_store if resolved_store
       end
 
       # Retry/visibility policy is settable so a Queue can propagate its own
@@ -49,7 +66,7 @@ module Tina4
         @mutex.synchronize do
           topic_dir = topic_path(message.topic)
           FileUtils.mkdir_p(topic_dir)
-          path = File.join(topic_dir, "#{message.id}.json")
+          path = job_file(topic_dir, message.id)
           File.write(path, message.to_json)
         end
       end
@@ -98,7 +115,7 @@ module Tina4
           return nil unless Dir.exist?(dir)
 
           target = id.to_s
-          Dir.glob(File.join(dir, "*.json")).each do |f|
+          job_files(dir).each do |f|
             data = JSON.parse(File.read(f))
             next unless data["id"].to_s == target
 
@@ -155,7 +172,7 @@ module Tina4
       end
 
       def dead_letter(message)
-        path = File.join(@dead_letter_dir, "#{message.id}.json")
+        path = job_file(@dead_letter_dir, message.id)
         data = message.to_hash
         data[:status] = "dead"
         File.write(path, JSON.generate(data))
@@ -164,14 +181,23 @@ module Tina4
       def size(topic)
         dir = topic_path(topic)
         return 0 unless Dir.exist?(dir)
-        Dir.glob(File.join(dir, "*.json")).length
+        job_files(dir).length
       end
 
+      # No-op: the file backend holds no connection to release.
+      #
+      # It exists so Queue#close can call ONE method on every backend instead of
+      # testing for it, and so switching TINA4_QUEUE_BACKEND to "lite" never
+      # turns a working close into a NoMethodError. It was the ONLY backend
+      # without one, so every `close if respond_to?(:close)` guard in the tree
+      # silently did nothing here. Idempotent by construction - nothing to drop.
+      def close; end
+
       # Count currently-reserved (in-flight) jobs for a topic.
       def reserved_count(topic)
         dir = reserved_path(topic)
         return 0 unless Dir.exist?(dir)
-        Dir.glob(File.join(dir, "*.json")).length
+        job_files(dir).length
       end
 
       # Count dead-letter / failed messages for a topic.
@@ -179,7 +205,7 @@ module Tina4
         return 0 unless Dir.exist?(@dead_letter_dir)
 
         count = 0
-        Dir.glob(File.join(@dead_letter_dir, "*.json")).each do |file|
+        job_files(@dead_letter_dir).each do |file|
           data = JSON.parse(File.read(file))
           count += 1 if data["topic"] == topic.to_s
         rescue JSON::ParserError
@@ -191,7 +217,7 @@ module Tina4
       def topics
         return [] unless Dir.exist?(@dir)
         Dir.children(@dir)
-           .reject { |d| d == "dead_letter" }
+           .reject { |d| d == Tina4::Queue::DEAD_LETTER_DIRNAME }
            .select { |d| File.directory?(File.join(@dir, d)) }
       end
 
@@ -200,7 +226,7 @@ module Tina4
       def dead_letters(topic, max_retries: 3)
         return [] unless Dir.exist?(@dead_letter_dir)
 
-        files = Dir.glob(File.join(@dead_letter_dir, "*.json")).sort_by { |f| File.mtime(f) }
+        files = job_files(@dead_letter_dir).sort_by { |f| File.mtime(f) }
         jobs = []
 
         files.each do |file|
@@ -225,7 +251,7 @@ module Tina4
         if dead_status?(status)
           return 0 unless Dir.exist?(@dead_letter_dir)
 
-          Dir.glob(File.join(@dead_letter_dir, "*.json")).each do |file|
+          job_files(@dead_letter_dir).each do |file|
             data = JSON.parse(File.read(file))
             if data["topic"] == topic.to_s
               File.delete(file)
@@ -238,7 +264,7 @@ module Tina4
           dir = topic_path(topic)
           return 0 unless Dir.exist?(dir)
 
-          Dir.glob(File.join(dir, "*.json")).each do |file|
+          job_files(dir).each do |file|
             data = JSON.parse(File.read(file))
             if data["status"].to_s == status.to_s
               File.delete(file)
@@ -261,7 +287,7 @@ module Tina4
         FileUtils.mkdir_p(dir)
         count = 0
 
-        Dir.glob(File.join(@dead_letter_dir, "*.json")).each do |file|
+        job_files(@dead_letter_dir).each do |file|
           data = JSON.parse(File.read(file))
           next unless data["topic"] == topic.to_s
           next if (data["attempts"] || 0) >= max_retries
@@ -290,7 +316,7 @@ module Tina4
         count = 0
         [topic_path(topic), reserved_path(topic)].each do |dir|
           next unless Dir.exist?(dir)
-          Dir.glob(File.join(dir, "*.json")).each do |file|
+          job_files(dir).each do |file|
             File.delete(file)
             count += 1
           rescue Errno::ENOENT
@@ -310,7 +336,7 @@ module Tina4
         dir = topic_path(topic)
         return [] unless Dir.exist?(dir)
         jobs = []
-        Dir.glob(File.join(dir, "*.json")).sort_by { |f| File.mtime(f) }.each do |file|
+        job_files(dir).sort_by { |f| File.mtime(f) }.each do |file|
           data = JSON.parse(File.read(file))
           attempts = data["attempts"] || 0
           next unless attempts > 0 && attempts < max_retries
@@ -333,7 +359,7 @@ module Tina4
         available_at = delay_seconds > 0 ? Time.now + delay_seconds : nil
         count = 0
 
-        Dir.glob(File.join(@dead_letter_dir, "*.json")).each do |file|
+        job_files(@dead_letter_dir).each do |file|
           data = JSON.parse(File.read(file))
           next unless data["topic"] == topic.to_s
           next if job_id && data["id"] != job_id.to_s
@@ -376,7 +402,7 @@ module Tina4
         now = Time.now
         candidates = []
 
-        Dir.glob(File.join(dir, "*.json")).each do |f|
+        job_files(dir).each do |f|
           data = JSON.parse(File.read(f))
           # Skip messages that are not yet available (delayed).
           if data["available_at"]
@@ -434,7 +460,7 @@ module Tina4
         data[:available_at] = available_at if available_at
         topic_dir = topic_path(job.topic)
         FileUtils.mkdir_p(topic_dir)
-        File.write(File.join(topic_dir, "#{job.id}.json"), JSON.generate(data))
+        File.write(job_file(topic_dir, job.id), JSON.generate(data))
       end
 
       # Move a failed job to the dead-letter directory. Terminal until a manual
@@ -456,12 +482,82 @@ module Tina4
           failed_at: Time.now.iso8601(6)
         }
         FileUtils.mkdir_p(@dead_letter_dir)
-        File.write(File.join(@dead_letter_dir, "#{job.id}.json"), JSON.generate(data))
+        File.write(job_file(@dead_letter_dir, job.id), JSON.generate(data))
+      end
+
+      # One place decides what a job file is called and where a topic lives, so
+      # no call site here can be left addressing a directory or an extension
+      # the dev-admin panel does not also read.
+      def job_files(dir)
+        Tina4::Queue.job_files(dir)
+      end
+
+      def job_file(dir, id)
+        Tina4::Queue.job_file(dir, id)
       end
 
       def topic_path(topic)
-        safe_topic = topic.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
-        File.join(@dir, safe_topic)
+        File.join(@dir, Tina4::Queue.topic_dirname(topic))
+      end
+
+      # Rescue a pre-alignment store, once, on the way past.
+      #
+      # Until this change the Ruby file backend stored jobs at
+      # Dir.pwd/.queue/<topic>/<id>.json and read TINA4_QUEUE_PATH NOWHERE, so
+      # EVERY existing Ruby app has its jobs there — including apps that had
+      # TINA4_QUEUE_PATH set, because the variable did nothing at all. Moving
+      # to the canonical <TINA4_QUEUE_PATH|data/queue>/<topic>/*.queue-data
+      # layout without this would strand all of them, and a stranded job is
+      # pending work somebody is still waiting on: data loss, not cosmetics.
+      #
+      # MOVE, never copy — a copy would leave the same job in two stores and
+      # let it be delivered twice. Each file is moved individually and never
+      # over an existing destination, so concurrent processes can each only win
+      # a given file once and neither can overwrite a job the other placed.
+      # FileUtils.mv (not File.rename) because the new store may be on another
+      # filesystem, which is the whole point of TINA4_QUEUE_PATH.
+      #
+      # The trigger is the legacy directory EXISTING, not the new one being
+      # absent: during a rolling upgrade an old instance can still be writing
+      # to .queue after the new store exists, and those jobs must be collected
+      # too. Emptied directories are pruned so the steady-state cost is the one
+      # Dir.exist? below.
+      def migrate_legacy_store
+        legacy_root = File.expand_path(File.join(Dir.pwd, LEGACY_DIRNAME))
+        return unless Dir.exist?(legacy_root)
+        return if legacy_root == File.expand_path(@dir)
+
+        moved = 0
+        Dir.glob(File.join(legacy_root, "**", "*#{LEGACY_JOB_EXTENSION}")).sort.each do |source|
+          relative = source.delete_prefix("#{legacy_root}#{File::SEPARATOR}")
+          destination = File.join(@dir, "#{relative.delete_suffix(LEGACY_JOB_EXTENSION)}#{JOB_EXTENSION}")
+          next if File.exist?(destination)
+
+          FileUtils.mkdir_p(File.dirname(destination))
+          FileUtils.mv(source, destination)
+          moved += 1
+        rescue SystemCallError
+          next # another process won the race, or the file is not ours to move
+        end
+
+        prune_empty_dirs(legacy_root)
+        return if moved.zero?
+
+        Tina4::Log.info(
+          "Queue: moved #{moved} job(s) from the legacy #{LEGACY_DIRNAME}/ store to #{@dir} " \
+          "(now #{JOB_EXTENSION} files, matching Python/PHP/Node)"
+        )
+      end
+
+      # Remove +root+ and every directory under it that is empty, deepest
+      # first, so the migration's fast path stops firing once it has nothing
+      # left to do. Anything still holding a file is left exactly as it is.
+      def prune_empty_dirs(root)
+        Dir.glob(File.join(root, "**", "*"))
+           .select { |path| File.directory?(path) }
+           .sort_by { |path| -path.length }
+           .each { |path| Dir.rmdir(path) rescue nil }
+        Dir.rmdir(root) rescue nil
       end
 
       # Directory holding a topic's reservation records (in-flight jobs).
@@ -490,12 +586,12 @@ module Tina4
         }
         dir = reserved_path(topic)
         FileUtils.mkdir_p(dir)
-        File.write(File.join(dir, "#{record[:id]}.json"), JSON.generate(record))
+        File.write(job_file(dir, record[:id]), JSON.generate(record))
       end
 
       # Delete a job's reservation record (best-effort).
       def clear_reservation(topic, id)
-        File.delete(File.join(reserved_path(topic), "#{id}.json"))
+        File.delete(job_file(reserved_path(topic), id))
       rescue Errno::ENOENT
         nil
       end
@@ -513,7 +609,7 @@ module Tina4
         return unless Dir.exist?(dir)
 
         now = Time.now
-        Dir.glob(File.join(dir, "*.json")).each do |file|
+        job_files(dir).each do |file|
           data = JSON.parse(File.read(file))
           available_at = data["available_at"] ? Time.parse(data["available_at"]) : now
           next if available_at > now # reservation still valid

data/lib/tina4/queue_backends/mongo_backend.rb

@@ -58,12 +58,22 @@ module Tina4
       end
 
       def enqueue(message)
+        # Queue#push already resolved delay_seconds into an available_at (nil when
+        # undelayed). Persisting it is what makes a delayed job invisible until
+        # its time: dequeue filters on it below. Before 3.13.95 this field was
+        # never written, so a delayed job on Mongo fired immediately while the
+        # same code delayed correctly on the file backend.
         collection.insert_one(
           _id: message.id,
           topic: message.topic,
           payload: message.payload,
           created_at: message.created_at.utc,
+          available_at: message.available_at&.utc,
           attempts: message.attempts,
+          # Stored so the dequeue sort can order on it. It was read back on the
+          # way out (doc["priority"] || 0) but never written, so every job
+          # scored 0 and the queue was pure FIFO.
+          priority: message.priority,
           status: "pending"
         )
       end
@@ -77,14 +87,27 @@ module Tina4
         # reserved_at so reclaim_expired can return the job if the consumer dies
         # before acknowledge/complete. This is the fix for the "reserved forever"
         # bug — previously available_at was left unchanged.
+        # A job is claimable only once available_at has passed. The $or arm is not
+        # optional: documents enqueued before available_at was written have no
+        # such field, and a bare { "$lte" => now } would strand every one of them
+        # in the collection forever.
         doc = collection.find_one_and_update(
-          { topic: topic, status: "pending" },
+          { topic: topic, status: "pending",
+            "$or" => [
+              { available_at: nil },
+              { available_at: { "$exists" => false } },
+              { available_at: { "$lte" => now } }
+            ] },
           { "$set" => {
             status: "processing",
             reserved_at: now,
             available_at: now + (@visibility_timeout || 0)
           } },
-          sort: { created_at: 1 },
+          # Highest priority first, ties broken oldest-first — the same ordering
+          # policy LiteBackend applies. Sorting on created_at alone made the
+          # backend pure FIFO, so an urgent job queued behind a backlog waited
+          # for all of it.
+          sort: { priority: -1, created_at: 1 },
           return_document: :after
         )
         return nil unless doc
@@ -162,7 +185,12 @@ module Tina4
         if job.attempts >= @max_retries
           collection.find_one_and_update(
             { _id: job.id },
+            # attempts MUST be persisted here. fail() increments the in-memory
+            # counter only; without writing it the dead-letter document keeps
+            # the value it held BEFORE the final attempt, so dead_letters()
+            # under-reported by one on every Mongo dead letter.
             { "$set" => { status: "dead", topic: "#{job.topic}.dead_letter",
+                          attempts: job.attempts,
                           error: error, reserved_at: nil } },
             upsert: true
           )
@@ -208,18 +236,59 @@ module Tina4
         )
       end
 
+      # Remove every pending job for a topic. Queue#clear used to return 0
+      # silently here, because this backend had no clear at all and the caller
+      # guarded on respond_to?(:clear) - so clearing a mongo-backed queue was a
+      # no-op that looked like success.
+      def clear(topic)
+        result = collection.delete_many(topic: topic, status: "pending")
+        result.deleted_count
+      end
+
+      # Claim ONE specific job by id, the same way dequeue claims the head.
+      # Queue#pop_by_id used to return nil silently for the same reason.
+      def find_by_id(topic, id)
+        now = Time.now.utc
+        doc = collection.find_one_and_update(
+          { _id: id, topic: topic, status: "pending" },
+          { "$set" => { status: "processing", reserved_at: now,
+                        available_at: now + (@visibility_timeout || 0) } },
+          return_document: :after
+        )
+        return nil unless doc
+
+        Tina4::Job.new(
+          topic: doc["topic"],
+          payload: doc["payload"],
+          id: doc["_id"],
+          priority: doc["priority"] || 0,
+          attempts: doc["attempts"] || 0
+        )
+      end
+
       def size(topic)
         collection.count_documents(topic: topic, status: "pending")
       end
 
+      # Jobs that failed but are still eligible for retry (under max_retries).
+      #
+      # Found by the ATTEMPTS COUNTER, not by a "failed" status. fail() under
+      # max_retries re-queues the job as "pending" (that is what makes the next
+      # dequeue redeliver it), so no document ever carries status="failed" on
+      # the normal path. This method did not exist at all, and Queue#failed
+      # silently returned [] for it — indistinguishable from "nothing has
+      # failed" (ADR-0022 decision 7).
+      def failed(topic, max_retries: 3)
+        collection.find(
+          topic: topic,
+          status: { "$in" => %w[pending failed] },
+          attempts: { "$gt" => 0, "$lt" => max_retries }
+        ).map { |doc| job_from_doc(doc) }
+      end
+
       def dead_letters(topic, max_retries: 3)
-        collection.find(topic: "#{topic}.dead_letter", status: "dead").map do |doc|
-          Tina4::Job.new(
-            topic: doc["topic"],
-            payload: doc["payload"],
-            id: doc["_id"]
-          )
-        end
+        collection.find(topic: "#{topic}.dead_letter", status: "dead")
+                  .map { |doc| job_from_doc(doc) }
       end
 
       def purge(topic, status)
@@ -227,24 +296,91 @@ module Tina4
         result.deleted_count
       end
 
+      # Re-queue failed-but-retryable jobs back to pending. Returns the count.
+      #
+      # Matched on the ATTEMPTS COUNTER for the same reason failed() is: the
+      # old query was status="failed", which requeue_with_error never writes,
+      # so this matched nothing and always returned 0 — silently reporting that
+      # there was nothing to retry.
       def retry_failed(topic, max_retries: 3)
         result = collection.update_many(
-          { topic: topic, status: "failed", attempts: { "$lt" => max_retries } },
+          { topic: topic,
+            # BOTH statuses. "pending" is where the auto-retry fail() path
+            # leaves a still-retryable job (it re-queues rather than marking
+            # it failed), and matching only "failed" is why this used to
+            # return 0 on every real failure. "failed" is still matched
+            # because reject(requeue: false) writes it, so a job explicitly
+            # parked as failed is retryable too.
+            status: { "$in" => %w[pending failed] },
+            attempts: { "$gt" => 0, "$lt" => max_retries } },
           # Reset available_at so re-queued failed jobs are visible again — they
           # were reserved with available_at in the future at dequeue. Clear
           # reserved_at too. (Same Bug B reason as requeue/fail.)
+          # status MUST be set back to "pending". The match now also picks up
+          # docs explicitly parked as "failed" (reject(requeue: false)), and
+          # those have to be flipped back or retry_failed reports a re-queue it
+          # never performed. A doc already pending is unaffected.
           { "$set" => { status: "pending", error: nil, reserved_at: nil,
                         available_at: requeue_available_at(@retry_backoff) } }
         )
         result.modified_count
       end
 
+      # Move ONE dead-lettered job back to its main topic as pending.
+      # Returns true when a job was revived, false when the id was not found.
+      def retry_job(topic, job_id: nil, delay_seconds: 0)
+        filter = { topic: "#{topic}.dead_letter", status: "dead" }
+        filter[:_id] = job_id if job_id
+        doc = collection.find(filter).first
+        return false unless doc
+
+        available = delay_seconds.to_f > 0 ? (Time.now.utc + delay_seconds.to_f) : Time.now.utc
+        collection.update_one(
+          { _id: doc["_id"] },
+          { "$set" => { topic: topic, status: "pending", error: nil,
+                        reserved_at: nil, available_at: available } }
+        )
+        true
+      end
+
+      # Close the MongoClient and release its connection pool.
+      #
+      # IDEMPOTENT by construction: the handles are dropped in an ensure, so a
+      # second close finds nothing and returns. Before 3.13.95 the ivars were
+      # left set, so a shutdown path that ran twice (an explicit close plus an
+      # at_exit / ensure) closed an already-closed client.
       def close
         @client&.close
+      ensure
+        @client = nil
+        @db = nil
       end
 
       private
 
+      # A stored document as a plain Hash with string keys, CARRYING attempts
+      # and error.
+      #
+      # Hash, not Job: the lite backend returns plain hashes from failed() and
+      # dead_letters(), so returning Job objects here would make the same
+      # caller need two different accessors depending on the backend — the very
+      # divergence this invariant exists to remove.
+      #
+      # The old dead_letters() mapped only topic/payload/id, so every Mongo dead
+      # letter read back as attempts=0 with no error text while the file backend
+      # reported the real values — a handler logging "died after N attempts:
+      # <reason>" printed "died after 0 attempts:" on Mongo only.
+      def job_from_doc(doc)
+        {
+          "id" => doc["_id"],
+          "topic" => doc["topic"],
+          "payload" => doc["payload"],
+          "attempts" => doc["attempts"] || 0,
+          "status" => doc["status"],
+          "error" => doc["error"]
+        }
+      end
+
       # Re-queue a failed job to pending, carrying the failure reason and
       # resetting available_at (now, or now + retry_backoff) + clearing
       # reserved_at so the next dequeue picks it up.