hirefire-resource 0.10.1 → 1.0.0

data/lib/hirefire/macro/deprecated/sidekiq.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module HireFire
+  module Macro
+    module Deprecated
+      # Provides backward compatibility with the deprecated Sidekiq macro.
+      # For new implementations, refer to {HireFire::Macro::Sidekiq}.
+      module Sidekiq
+        # Calculates the latency (in seconds) for the specified Sidekiq queue.
+        #
+        # The method uses the Sidekiq::Queue class to obtain the latency of a queue, which
+        # is the duration since the oldest job in the queue was enqueued.
+        #
+        # @param queue [String, Symbol] The name of the queue to measure latency.
+        #   Defaults to "default" if no queue name is provided.
+        # @return [Float] The latency of the queue in seconds.
+        # @example Calculating latency for the default queue
+        #   HireFire::Macro::Sidekiq.latency
+        # @example Calculating latency for the "critical" queue
+        #   HireFire::Macro::Sidekiq.latency("critical")
+        def latency(queue = "default")
+          ::Sidekiq::Queue.new(queue.to_s).latency
+        end
+
+        # Counts the number of jobs in the specified Sidekiq queue(s).
+        #
+        # The method supports various options to include or exclude jobs from
+        # specific sets like scheduled, retries, or in-progress jobs.
+        #
+        # @param args [Array<String, Symbol, Hash>] Queue names to count jobs in and an optional hash of options.
+        #   Pass an empty array or no arguments to count jobs in all queues.
+        #   The last argument can be a Hash of options to modify the count behavior.
+        #   Possible keys are :skip_scheduled, :skip_retries, :skip_working.
+        #   These keys are booleans which default to false.
+        # @return [Integer] Total number of jobs in the specified queues.
+        # @example Counting jobs in all queues
+        #   HireFire::Macro::Sidekiq.queue
+        # @example Counting jobs in the "default" and "critical" queues
+        #   HireFire::Macro::Sidekiq.queue("default", "critical")
+        # @example Counting jobs in the "default" queue, excluding scheduled jobs
+        #   HireFire::Macro::Sidekiq.queue("default", skip_scheduled: true)
+        # @example Counting jobs in the "default" queue, excluding retryable jobs
+        #   HireFire::Macro::Sidekiq.queue("default", skip_retries: true)
+        # @example Counting jobs in the "default" queue, excluding in-progress jobs
+        #   HireFire::Macro::Sidekiq.queue("default", skip_working: true)
+        def queue(*args)
+          require "sidekiq/api"
+
+          args.flatten!
+
+          options = args.last.is_a?(Hash) ? args.pop : {}
+          queues = args.map(&:to_s)
+          all_queues = ::Sidekiq::Queue.all.map(&:name)
+          queues = all_queues if queues.empty?
+
+          if fast_lookup_capable?(queues, all_queues)
+            fast_lookup(options)
+          else
+            dynamic_lookup(queues, options)
+          end
+        end
+
+        private
+
+        def fast_lookup_capable?(queues, all_queues)
+          # When no queue names are provided (or all of them are), we know we
+          # can perform much faster counts using Sidekiq::Stats and Redis
+          queues.sort == all_queues.sort
+        end
+
+        def fast_lookup(options)
+          stats = ::Sidekiq::Stats.new
+
+          in_queues = stats.enqueued
+
+          if !options[:skip_scheduled]
+            in_schedule = ::Sidekiq.redis { |c| c.zcount("schedule", "-inf", Time.now.to_f) }
+          end
+
+          if !options[:skip_retries]
+            in_retry = ::Sidekiq.redis { |c| c.zcount("retry", "-inf", Time.now.to_f) }
+          end
+
+          if !options[:skip_working]
+            in_progress = stats.workers_size
+          end
+
+          [in_queues, in_schedule, in_retry, in_progress].compact.inject(&:+)
+        end
+
+        def dynamic_lookup(queues, options)
+          in_queues = queues.inject(0) do |memo, name|
+            memo += ::Sidekiq::Queue.new(name).size
+            memo
+          end
+
+          if !options[:skip_scheduled]
+            max = options[:max_scheduled]
+
+            # For potentially long-running loops, compare all jobs against
+            # time when the set snapshot was taken to avoid incorrect counts.
+            now = Time.now
+
+            in_schedule = ::Sidekiq::ScheduledSet.new.inject(0) do |memo, job|
+              memo += 1 if queues.include?(job["queue"]) && job.at <= now
+              break memo if max && memo >= max
+              memo
+            end
+          end
+
+          if !options[:skip_retries]
+            now = Time.now
+
+            in_retry = ::Sidekiq::RetrySet.new.inject(0) do |memo, job|
+              memo += 1 if queues.include?(job["queue"]) && job.at <= now
+              memo
+            end
+          end
+
+          now = Time.now
+          now_as_i = now.to_i
+
+          if !options[:skip_working]
+            in_progress = ::Sidekiq::Workers.new.count do |key, tid, job|
+              if Gem::Version.new(::Sidekiq::VERSION) >= Gem::Version.new("7.0.0")
+                queues.include?(job.queue) && job.run_at <= now
+              else
+                queues.include?(job["queue"]) && job["run_at"] <= now_as_i
+              end
+            end
+          end
+
+          [in_queues, in_schedule, in_retry, in_progress].compact.inject(&:+)
+        end
+      end
+    end
+  end
+end

data/lib/hirefire/macro/good_job.rb

@@ -1,25 +1,60 @@
 # frozen_string_literal: true
 
+require_relative "deprecated/good_job"
+
 module HireFire
   module Macro
     module GoodJob
+      extend HireFire::Macro::Deprecated::GoodJob
+      extend HireFire::Utility
       extend self
 
-      # Queries the PostgreSQL database through GoodJob in order to
-      # count the amount of jobs in the specified queue.
-      #
-      # @example Queue Macro Usage
-      #   HireFire::Macro::GoodJob.queue # counts all queues.
-      #   HireFire::Macro::GoodJob.queue("email") # counts the `email` queue.
+      # Calculates the maximum job queue latency using GoodJob. If no queues are specified, it
+      # measures latency across all available queues.
       #
-      # @param [String] queue the queue name to count. (default: nil # gets all queues)
-      # @return [Integer] the number of jobs in the queue(s).
+      # @param queues [Array<String, Symbol>] (optional) Names of the queues for latency
+      #   measurement. If not provided, latency is measured across all queues.
+      # @return [Float] Maximum job queue latency in seconds.
+      # @example Calculate latency across all queues
+      #   HireFire::Macro::GoodJob.job_queue_latency
+      # @example Calculate latency for the "default" queue
+      #   HireFire::Macro::GoodJob.job_queue_latency(:default)
+      # @example Calculate latency across "default" and "mailer" queues
+      #   HireFire::Macro::GoodJob.job_queue_latency(:default, :mailer)
+      def job_queue_latency(*queues)
+        queues = normalize_queues(queues, allow_empty: true)
+        query = ::GoodJob::Execution
+        query = query.where(queue_name: queues) if queues.any?
+        query = query.where(finished_at: nil)
+        query = query.where(scheduled_at: ..Time.now).or(query.where(scheduled_at: nil))
+        query = query.order(scheduled_at: :asc, created_at: :asc)
+
+        if (job = query.first)
+          Time.now - (job.scheduled_at || job.created_at)
+        else
+          0.0
+        end
+      end
+
+      # Calculates the total job queue size using GoodJob. If no queues are specified, it
+      # measures size across all available queues.
       #
-      def queue(*queues)
-        base_class = defined?(::GoodJob::Execution) ? ::GoodJob::Execution : ::GoodJob::Job
-        scope = base_class.only_scheduled.unfinished
-        scope = scope.where(queue_name: queues) if queues.any?
-        scope.count
+      # @param queues [Array<String, Symbol>] (optional) Names of the queues for size measurement.
+      #   If not provided, size is measured across all queues.
+      # @return [Integer] Total job queue size.
+      # @example Calculate size across all queues
+      #   HireFire::Macro::GoodJob.job_queue_size
+      # @example Calculate size for the "default" queue
+      #   HireFire::Macro::GoodJob.job_queue_size(:default)
+      # @example Calculate size across "default" and "mailer" queues
+      #   HireFire::Macro::GoodJob.job_queue_size(:default, :mailer)
+      def job_queue_size(*queues)
+        queues = normalize_queues(queues, allow_empty: true)
+        query = ::GoodJob::Execution
+        query = query.where(queue_name: queues) if queues.any?
+        query = query.where(finished_at: nil)
+        query = query.where(scheduled_at: ..Time.now).or(query.where(scheduled_at: nil))
+        query.count
       end
     end
   end

data/lib/hirefire/macro/que.rb

@@ -1,55 +1,75 @@
 # frozen_string_literal: true
 
+require_relative "deprecated/que"
+
 module HireFire
   module Macro
     module Que
+      extend HireFire::Macro::Deprecated::Que
+      extend HireFire::Utility
       extend self
 
-      # Queries the PostgreSQL database through Que in order to
-      # count the amount of jobs in the specified queue.
-      #
-      # @example Queue Macro Usage
-      #   HireFire::Macro::Que.queue # counts all queues.
-      #   HireFire::Macro::Que.queue("email") # counts the `email` queue.
-      #
-      # @param [String] queue the queue name to count. (default: nil # gets all queues)
-      # @return [Integer] the number of jobs in the queue(s).
+      # Calculates the maximum job queue latency using Que. If no queues are specified, it
+      # measures latency across all available queues.
       #
-      def queue(*queues)
-        query = queues.empty? && base_query || base_query + " AND queue IN (#{names(queues)})"
-        results = ::Que.execute(query).first
-        (results[:total] || results["total"]).to_i
+      # @param queues [Array<String, Symbol>] (optional) Names of the queues for latency
+      #   measurement. If not provided, latency is measured across all queues.
+      # @return [Float] Maximum job queue latency in seconds.
+      # @example Calculate latency across all queues
+      #   HireFire::Macro::Que.job_queue_latency
+      # @example Calculate latency for the "default" queue
+      #   HireFire::Macro::Que.job_queue_latency(:default)
+      # @example Calculate latency across "default" and "mailer" queues
+      #   HireFire::Macro::Que.job_queue_latency(:default, :mailer)
+      def job_queue_latency(*queues)
+        query = <<~SQL
+          SELECT run_at FROM que_jobs
+          WHERE run_at <= NOW()
+          AND finished_at IS NULL
+          AND expired_at IS NULL
+          #{filter_by_queues_if_any(queues)}
+          ORDER BY run_at ASC LIMIT 1
+        SQL
+
+        result = ::Que.execute(query).first
+        result ? (Time.now - result[:run_at].to_time) : 0.0
       end
 
-      private
+      # Calculates the total job queue size using Que. If no queues are specified, it
+      # measures size across all available queues.
+      #
+      # @param queues [Array<String, Symbol>] (optional) Names of the queues for size measurement.
+      #   If not provided, size is measured across all queues.
+      # @return [Integer] Total job queue size.
+      # @example Calculate size across all queues
+      #   HireFire::Macro::Que.job_queue_size
+      # @example Calculate size for the "default" queue
+      #   HireFire::Macro::Que.job_queue_size(:default)
+      # @example Calculate size across "default" and "mailer" queues
+      #   HireFire::Macro::Que.job_queue_size(:default, :mailer)
+      def job_queue_size(*queues)
+        query = <<~SQL
+          SELECT COUNT(*) AS total FROM que_jobs
+          WHERE run_at <= NOW()
+          AND finished_at IS NULL
+          AND expired_at IS NULL
+          #{filter_by_queues_if_any(queues)}
+        SQL
 
-      def base_query
-        return QUE_V0_QUERY if defined?(::Que::Version)
-        return QUE_V1_QUERY if defined?(::Que::VERSION)
-        raise "Couldn't find Que version"
+        ::Que.execute(query).first.fetch(:total).to_i
       end
 
-      def names(queues)
-        queues.map { |queue| "'#{queue}'" }.join(",")
-      end
+      private
 
-      def query_const(query)
-        query.gsub(/\s+/, " ").strip.freeze
+      def filter_by_queues_if_any(queues)
+        queues = normalize_queues(queues, allow_empty: true)
+        queues = queues.map(&method(:sanitize_sql)).join(", ")
+        queues.empty? ? "" : "AND queue IN (#{queues})"
       end
 
-      QUE_V0_QUERY = query_const(<<-QUERY)
-        SELECT COUNT(*) AS total
-        FROM que_jobs
-        WHERE run_at < NOW()
-      QUERY
-
-      QUE_V1_QUERY = query_const(<<-QUERY)
-        SELECT COUNT(*) AS total
-        FROM que_jobs
-        WHERE finished_at IS NULL
-        AND expired_at IS NULL
-        AND run_at <= NOW()
-      QUERY
+      def sanitize_sql(value)
+        "'" + value.to_s.gsub(/['"\\]/, '\\\\\\&') + "'"
+      end
     end
   end
 end

data/lib/hirefire/macro/queue_classic.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative "deprecated/queue_classic"
+
+module HireFire
+  module Macro
+    module QC
+      extend HireFire::Macro::Deprecated::QC
+      extend HireFire::Utility
+      extend self
+
+      # Calculates the maximum job queue latency using Queue Classic. If no queues are specified, it
+      # measures latency across all available queues.
+      #
+      # @param queues [Array<String, Symbol>] (optional) Names of the queues for latency
+      #   measurement. If not provided, latency is measured across all queues.
+      # @return [Float] Maximum job queue latency in seconds.
+      # @example Calculate latency across all queues
+      #   HireFire::Macro::QC.job_queue_latency
+      # @example Calculate latency for the "default" queue
+      #   HireFire::Macro::QC.job_queue_latency(:default)
+      # @example Calculate latency across "default" and "mailer" queues
+      #   HireFire::Macro::QC.job_queue_latency(:default, :mailer)
+      def job_queue_latency(*queues)
+        queues = normalize_queues(queues, allow_empty: true)
+
+        query = <<~SQL
+          SELECT EXTRACT(EPOCH FROM (now() - scheduled_at)) AS latency
+          FROM #{::QC.table_name}
+          WHERE scheduled_at <= now()
+          #{filter_by_queues_if_any(queues)}
+          ORDER BY scheduled_at ASC
+          LIMIT 1
+        SQL
+
+        connection = ::QC.default_conn_adapter
+
+        result = if queues.any?
+          connection.execute(query, "{#{queues.to_a.join(",")}}")
+        else
+          connection.execute(query)
+        end
+
+        (result && result["latency"]) ? result["latency"].to_f : 0.0
+      end
+
+      # Calculates the total job queue size using Queue Classic. If no queues are specified, it
+      # measures size across all available queues.
+      #
+      # @param queues [Array<String, Symbol>] (optional) Names of the queues for size measurement.
+      #   If not provided, size is measured across all queues.
+      # @return [Integer] Total job queue size.
+      # @example Calculate size across all queues
+      #   HireFire::Macro::QC.job_queue_size
+      # @example Calculate size for the "default" queue
+      #   HireFire::Macro::QC.job_queue_size(:default)
+      # @example Calculate size across "default" and "mailer" queues
+      #   HireFire::Macro::QC.job_queue_size(:default, :mailer)
+      def job_queue_size(*queues)
+        queues = normalize_queues(queues, allow_empty: true)
+
+        query = <<~SQL
+          SELECT COUNT(*) FROM #{::QC.table_name}
+          WHERE scheduled_at <= now()
+          #{filter_by_queues_if_any(queues)}
+        SQL
+
+        connection = ::QC.default_conn_adapter
+
+        result = if queues.any?
+          connection.execute(query, "{#{queues.to_a.join(",")}}")
+        else
+          connection.execute(query)
+        end
+
+        result["count"].to_i
+      end
+
+      private
+
+      def filter_by_queues_if_any(queues)
+        queues.any? ? "AND q_name = ANY($1::text[])" : ""
+      end
+    end
+  end
+end

data/lib/hirefire/macro/resque.rb

@@ -1,43 +1,129 @@
 # frozen_string_literal: true
 
+require_relative "deprecated/resque"
+
 module HireFire
   module Macro
     module Resque
+      extend HireFire::Errors::JobQueueLatencyUnsupported
+      extend HireFire::Macro::Deprecated::Resque
+      extend HireFire::Utility
       extend self
 
-      # Counts the amount of jobs in the (provided) Resque queue(s).
-      #
-      # @example Resque Macro Usage
-      #   HireFire::Macro::Resque.queue # all queues
-      #   HireFire::Macro::Resque.queue("email") # only email queue
-      #   HireFire::Macro::Resque.queue("audio", "video") # audio and video queues
-      #
-      # @param [Array] queues provide one or more queue names, or none for "all".
-      # @return [Integer] the number of jobs in the queue(s).
+      SIZE_METHODS = [
+        :enqueued_size,
+        :working_size,
+        :scheduled_size
+      ].freeze
+
+      # Calculates the maximum job queue size using Resque. If no queues are specified, it
+      # measures size across all available queues.
       #
-      def queue(*queues)
-        queues = queues.flatten.map(&:to_s)
-        queues = ::Resque.queues if queues.empty?
+      # @param queues [Array<String, Symbol>] (optional) Names of the queues for size measurement.
+      #   If not provided, size is measured across all queues.
+      # @return [Integer] Total job queue size.
+      # @example Calculate size across all queues
+      #   HireFire::Macro::Resque.job_queue_size
+      # @example Calculate size for the "default" queue
+      #   HireFire::Macro::Resque.job_queue_size(:default)
+      # @example Calculate size across the "default" and "mailer" queues
+      #   HireFire::Macro::Resque.job_queue_size(:default, :mailer)
+      def job_queue_size(*queues)
+        queues = normalize_queues(queues, allow_empty: true)
+
+        SIZE_METHODS.sum do |size_method|
+          method(size_method).call(queues)
+        end
+      end
 
-        return 0 if queues.empty?
+      private
 
-        redis = ::Resque.redis
-        ids = Array(redis.smembers(:workers)).compact
-        raw_jobs = redis.pipelined do |redis|
-          ids.map { |id| redis.get("worker:#{id}") }
+      def enqueued_size(queues)
+        queues = registered_queues if queues.empty?
+
+        ::Resque.redis.pipelined do |pipeline|
+          queues.each do |queue|
+            pipeline.llen("queue:#{queue}")
+          end
+        end.sum
+      end
+
+      def working_size(queues)
+        ids = ::Resque.redis.smembers(:workers).compact
+
+        workers = ::Resque.redis.pipelined do |pipeline|
+          ids.each do |id|
+            pipeline.get("worker:#{id}")
+          end
+        end.compact
+
+        if queues.empty?
+          workers.count
+        else
+          workers.count do |worker|
+            queues.include?(::Resque.decode(worker)["queue"])
+          end
         end
-        jobs = raw_jobs.map { |raw_job| ::Resque.decode(raw_job) || {} }
+      end
 
-        in_queues = redis.pipelined do |redis|
-          queues.map { |queue| redis.llen("queue:#{queue}") }
-        end.map(&:to_i).inject(&:+)
+      def scheduled_size(queues)
+        cursor = 0
+        batch = 1000
+        total_size = 0
+        current_time = Time.now.to_i
 
-        in_progress = jobs.inject(0) do |memo, job|
-          memo += 1 if queues.include?(job["queue"])
-          memo
+        loop do
+          timestamps = ::Resque.redis.zrangebyscore(
+            "delayed_queue_schedule",
+            "-inf",
+            current_time,
+            limit: [cursor, batch]
+          )
+
+          break if timestamps.empty?
+
+          if queues.empty?
+            total_size += ::Resque.redis.pipelined do |pipeline|
+              timestamps.each do |timestamp|
+                pipeline.llen("delayed:#{timestamp}")
+              end
+            end.sum
+          else
+            timestamps.each do |timestamp|
+              job_cursor = 0
+
+              loop do
+                encoded_jobs = ::Resque.redis.lrange(
+                  "delayed:#{timestamp}",
+                  job_cursor,
+                  job_cursor + batch - 1
+                )
+
+                break if encoded_jobs.empty?
+
+                total_size += encoded_jobs.count do |encoded_job|
+                  queues.include?(::Resque.decode(encoded_job)["queue"])
+                end
+
+                break if encoded_jobs.size < batch
+
+                job_cursor += batch
+              end
+            end
+          end
+
+          break if timestamps.size < batch
+
+          cursor += batch
         end
 
-        in_queues + in_progress
+        total_size
+      end
+
+      private
+
+      def registered_queues
+        ::Resque.redis.keys("queue:*").map { |key| key[6..] }.to_set
       end
     end
   end