sidekiq-amigo 1.9.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 330ead7f8face652362d632c9c127aa57b298d5587d1da392769b0458d1f8915
-  data.tar.gz: 7c0b0b9b1658eecfd202ef294580417233421edf4233eef522b747ce9c411f52
+  metadata.gz: 3d3e3fb7d4b61921787b7e2baf6f6cdb25008877b9e33b786fc967b40bc9e305
+  data.tar.gz: 497eeba83785fe528b6053b6e1b8158a4459f21bdde74b7b056fe20a17a77e4e
 SHA512:
-  metadata.gz: 898c24b82de6313c33e50b58a2c4d64ca365918a0aee2b223b452fea9428cc4d265b15975ebc2f7c7e689f547898e65b7c52d3e50edbe9bcdb062f3b25f3f0b6
-  data.tar.gz: a5ffcb482859fcddd6727a9869bef3ed00c8edb5b1b44f0cde84b31eddb44c6f84fbac6b45d8b8fccf06c712f6792a8ce3e211b4a49083fe8c0bab76474a5d38
+  metadata.gz: 9c41c3a9b2483ba59ad03b7fbde4bad4c21bb7516f857e5e287805236372cc71f9a08c03da0ee5015286607d71e069b38de3310d6c13bf74b85e2509b7de08b8
+  data.tar.gz: 40ed4bd8c85cc532d6f31aa1bbeadb3c13fb949555d4f7e31099658bbd3021586466a0af6fe1bfefae102def7a0852fd57a4b72c57e6717415fee488f19edb1a

data/lib/amigo/memory_pressure.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Amigo
+  # Helper class to detect when the Redis server is under memory pressure.
+  # In these cases, we want to disable queue backoff behavior.
+  # There is a significant risk that the backoff behavior will take jobs from the queue,
+  # and immediately try and reschedule them. If that happens in an OOM condition,
+  # the re-push will fail and the job can be lost.
+  #
+  # Additionally, the backoff behavior causes delays that slow down the clearing of the queue.
+  #
+  # In these high-memory-utilization conditions, it makes more sense to disable the backoff logic
+  # and just brute force to try to get through the queue.
+  class MemoryPressure
+    # Percentage at which the server is considered under memory pressure.
+    DEFAULT_THRESHOLD = 90
+
+    # Default seconds a memory check is good for. See +check_ttl+.
+    DEFAULT_CHECK_TTL = 120
+
+    class << self
+      # Return the singleton instance, creating a cached value if needed.
+      def instance
+        return @instance ||= self.new
+      end
+
+      # Set the instance, or use nil to reset.
+      attr_writer :instance
+    end
+
+    # When did we last check for pressure?
+    attr_reader :last_checked_at
+
+    # What was the result of the last check?
+    # true is under pressure, false if not.
+    attr_reader :last_check_result
+
+    # See +DEFAULT_CHECK_TTL+.
+    attr_reader :check_ttl
+
+    # See +DEFAULT_THRESHOLD+.
+    attr_reader :threshold
+
+    def initialize(check_ttl: DEFAULT_CHECK_TTL, threshold: DEFAULT_THRESHOLD)
+      @last_checked_at = nil
+      @check_ttl = check_ttl
+      @threshold = threshold
+      @last_check_result = nil
+    end
+
+    # Return true if the server is under memory pressure.
+    # When this is the case, we want to disable backoff,
+    # since it will delay working through the queue,
+    # and can also result in a higher likelihood of lost jobs,
+    # since returning them back to the queue will fail.
+    def under_pressure?
+      return @last_check_result unless self.needs_check?
+      @last_check_result = self.calculate_under_pressure
+      @last_checked_at = Time.now
+      return @last_check_result
+    end
+
+    private def needs_check?
+      return true if @last_checked_at.nil?
+      return (@last_checked_at + @check_ttl) < Time.now
+    end
+
+    private def calculate_under_pressure
+      meminfo = self.get_memory_info
+      used_bytes = meminfo.fetch("used_memory", "0").to_f
+      max_bytes = meminfo.fetch("maxmemory", "0").to_f
+      return false if used_bytes.zero? || max_bytes.zero?
+      percentage = (used_bytes / max_bytes) * 100
+      return percentage > self.threshold
+    end
+
+    protected def get_memory_info
+      Sidekiq.redis do |c|
+        c.info :memory
+      end
+    end
+  end
+end

data/lib/amigo/queue_backoff_job.rb

@@ -3,6 +3,8 @@
 require "sidekiq"
 require "sidekiq/api"
 
+require "amigo/memory_pressure"
+
 # Queue backoff jobs are used for jobs that should not saturate workers,
 # such that jobs on dependent queues end up not running for a while.
 #
@@ -45,6 +47,11 @@ require "sidekiq/api"
 # This is a fast call (it just gets the last item), but it's not free,
 # so users should be aware of it.
 #
+# == High Memory Utilization
+#
+# Queue backoff behavior is automatically disabled under high memory utilization,
+# as per +Amigo::MemoryPressure+.
+#
 module Amigo
   module QueueBackoffJob
     def self.included(cls)
@@ -126,6 +133,7 @@ module Amigo
     module PrependedMethods
       def perform(*args)
         return super unless ::Amigo::QueueBackoffJob.enabled?
+        return super if ::Amigo::MemoryPressure.instance.under_pressure?
         # rubocop:disable Style/GuardClause, Lint/NonLocalExitFromIterator
         dependent_queues.each do |qname|
           latency = Amigo::QueueBackoffJob.check_latency(qname)

data/lib/amigo/semaphore_backoff_job.rb

@@ -2,9 +2,8 @@
 
 require "sidekiq"
 
-module Amigo
-  # This is a placeholder until it's migrated to Amigo proper
-end
+require "amigo"
+require "amigo/memory_pressure"
 
 # Semaphore backoff jobs can reschedule themselves to happen at a later time
 # if there is too high a contention on a semaphore.
@@ -48,6 +47,11 @@ end
 # around missing decrements), if we ever detect a negative 'jobs running',
 # we warn and remove the key entirely.
 #
+# == High Memory Utilization
+#
+# Queue backoff behavior is automatically disabled under high memory utilization,
+# as per +Amigo::MemoryPressure+.
+#
 module Amigo
   module SemaphoreBackoffJob
     def self.included(cls)
@@ -94,6 +98,7 @@ module Amigo
       def perform(*args)
         self.before_perform(*args) if self.respond_to?(:before_perform)
         return super unless ::Amigo::SemaphoreBackoffJob.enabled?
+        return super if ::Amigo::MemoryPressure.instance.under_pressure?
         key = self.semaphore_key
         size = self.semaphore_size
         # Create a simple counter for the semaphore key.

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Amigo
-  VERSION = "1.9.0"
+  VERSION = "1.10.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Lithic Technology
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-03 00:00:00.000000000 Z
+date: 2025-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -179,6 +179,7 @@ files:
 - lib/amigo/autoscaler/heroku.rb
 - lib/amigo/deprecated_jobs.rb
 - lib/amigo/job.rb
+- lib/amigo/memory_pressure.rb
 - lib/amigo/queue_backoff_job.rb
 - lib/amigo/rate_limited_error_handler.rb
 - lib/amigo/retry.rb