sidekiq-amigo 1.8.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a32299bc84d87b6c497414b4fcb6b89278e2993543eff1ba07bcf1a9fb75c59f
-  data.tar.gz: 27acf41f1e0235536da93d7dfd0b2a5ad1f65e4e993f428967e8072e7213f83f
+  metadata.gz: 3d3e3fb7d4b61921787b7e2baf6f6cdb25008877b9e33b786fc967b40bc9e305
+  data.tar.gz: 497eeba83785fe528b6053b6e1b8158a4459f21bdde74b7b056fe20a17a77e4e
 SHA512:
-  metadata.gz: 47e75ba71d67411a80e7d5ae20f06c0b3bb4e0fb056356e63fb47ec3cd84cdb72f0deb6e2bd00f5cc0cd091267dc30725e4607ae018b7f29a31093c4c9e82ebd
-  data.tar.gz: 761f7febdba270fe365f39cb740f734615118bd8c160485ac9296119f00124458f1970e159dbdd0ed8a6fbecd4c66df8bfe07f86803a14e2982e738fbe69115c
+  metadata.gz: 9c41c3a9b2483ba59ad03b7fbde4bad4c21bb7516f857e5e287805236372cc71f9a08c03da0ee5015286607d71e069b38de3310d6c13bf74b85e2509b7de08b8
+  data.tar.gz: 40ed4bd8c85cc532d6f31aa1bbeadb3c13fb949555d4f7e31099658bbd3021586466a0af6fe1bfefae102def7a0852fd57a4b72c57e6717415fee488f19edb1a

data/lib/amigo/autoscaler/heroku.rb

@@ -81,7 +81,6 @@ module Amigo
         app_id_or_app_name: ENV.fetch("HEROKU_APP_NAME"),
         formation_id_or_formation_type: "worker"
       )
-
         @heroku = heroku
         @max_additional_workers = max_additional_workers
         @app_id_or_app_name = app_id_or_app_name

data/lib/amigo/autoscaler.rb

@@ -92,6 +92,13 @@ module Amigo
     # Proc/callable called with (level, message, params={}).
     # By default, use +Amigo.log+ (which logs to the Sidekiq logger).
     attr_reader :log
+    # Proc called with an exception that occurs while the thread is running.
+    # If the handler returns +true+, then the thread will keep going.
+    # All other values will kill the thread, which breaks autoscaling.
+    # Note that Amigo automatically logs unhandled exceptions at :error level.
+    # If you use an error reporter like Sentry, you can pass in something like:
+    #   -> (e) { Sentry.capture_exception(e) }
+    attr_reader :on_unhandled_exception
 
     def initialize(
       poll_interval: 20,
@@ -101,9 +108,9 @@ module Amigo
       alert_interval: 120,
       latency_restored_threshold: latency_threshold,
       latency_restored_handlers: [:log],
-      log: ->(level, message, params={}) { Amigo.log(nil, level, message, params) }
+      log: ->(level, message, params={}) { Amigo.log(nil, level, message, params) },
+      on_unhandled_exception: nil
     )
-
       raise ArgumentError, "latency_threshold must be > 0" if
         latency_threshold <= 0
       raise ArgumentError, "latency_restored_threshold must be >= 0" if
@@ -118,8 +125,10 @@ module Amigo
       @latency_restored_threshold = latency_restored_threshold
       @latency_restored_handlers = latency_restored_handlers.freeze
       @log = log
+      @on_unhandled_exception = on_unhandled_exception
     end
 
+    # @return [Thread]
     def polling_thread
       return @polling_thread
     end
@@ -189,6 +198,14 @@ module Amigo
     end
 
     def check
+      self._check
+    rescue StandardError => e
+      self._log(:error, "async_autoscaler_unhandled_error", exception: e)
+      handled = self.on_unhandled_exception&.call(e)
+      raise e unless handled.eql?(true)
+    end
+
+    def _check
       now = Time.now
       skip_check = now < (@last_alerted + self.alert_interval)
       if skip_check

data/lib/amigo/memory_pressure.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Amigo
+  # Helper class to detect when the Redis server is under memory pressure.
+  # In these cases, we want to disable queue backoff behavior.
+  # There is a significant risk that the backoff behavior will take jobs from the queue,
+  # and immediately try and reschedule them. If that happens in an OOM condition,
+  # the re-push will fail and the job can be lost.
+  #
+  # Additionally, the backoff behavior causes delays that slow down the clearing of the queue.
+  #
+  # In these high-memory-utilization conditions, it makes more sense to disable the backoff logic
+  # and just brute force to try to get through the queue.
+  class MemoryPressure
+    # Percentage at which the server is considered under memory pressure.
+    DEFAULT_THRESHOLD = 90
+
+    # Default seconds a memory check is good for. See +check_ttl+.
+    DEFAULT_CHECK_TTL = 120
+
+    class << self
+      # Return the singleton instance, creating a cached value if needed.
+      def instance
+        return @instance ||= self.new
+      end
+
+      # Set the instance, or use nil to reset.
+      attr_writer :instance
+    end
+
+    # When did we last check for pressure?
+    attr_reader :last_checked_at
+
+    # What was the result of the last check?
+    # true is under pressure, false if not.
+    attr_reader :last_check_result
+
+    # See +DEFAULT_CHECK_TTL+.
+    attr_reader :check_ttl
+
+    # See +DEFAULT_THRESHOLD+.
+    attr_reader :threshold
+
+    def initialize(check_ttl: DEFAULT_CHECK_TTL, threshold: DEFAULT_THRESHOLD)
+      @last_checked_at = nil
+      @check_ttl = check_ttl
+      @threshold = threshold
+      @last_check_result = nil
+    end
+
+    # Return true if the server is under memory pressure.
+    # When this is the case, we want to disable backoff,
+    # since it will delay working through the queue,
+    # and can also result in a higher likelihood of lost jobs,
+    # since returning them back to the queue will fail.
+    def under_pressure?
+      return @last_check_result unless self.needs_check?
+      @last_check_result = self.calculate_under_pressure
+      @last_checked_at = Time.now
+      return @last_check_result
+    end
+
+    private def needs_check?
+      return true if @last_checked_at.nil?
+      return (@last_checked_at + @check_ttl) < Time.now
+    end
+
+    private def calculate_under_pressure
+      meminfo = self.get_memory_info
+      used_bytes = meminfo.fetch("used_memory", "0").to_f
+      max_bytes = meminfo.fetch("maxmemory", "0").to_f
+      return false if used_bytes.zero? || max_bytes.zero?
+      percentage = (used_bytes / max_bytes) * 100
+      return percentage > self.threshold
+    end
+
+    protected def get_memory_info
+      Sidekiq.redis do |c|
+        c.info :memory
+      end
+    end
+  end
+end

data/lib/amigo/queue_backoff_job.rb

@@ -3,6 +3,8 @@
 require "sidekiq"
 require "sidekiq/api"
 
+require "amigo/memory_pressure"
+
 # Queue backoff jobs are used for jobs that should not saturate workers,
 # such that jobs on dependent queues end up not running for a while.
 #
@@ -45,6 +47,11 @@ require "sidekiq/api"
 # This is a fast call (it just gets the last item), but it's not free,
 # so users should be aware of it.
 #
+# == High Memory Utilization
+#
+# Queue backoff behavior is automatically disabled under high memory utilization,
+# as per +Amigo::MemoryPressure+.
+#
 module Amigo
   module QueueBackoffJob
     def self.included(cls)
@@ -126,6 +133,7 @@ module Amigo
     module PrependedMethods
       def perform(*args)
         return super unless ::Amigo::QueueBackoffJob.enabled?
+        return super if ::Amigo::MemoryPressure.instance.under_pressure?
         # rubocop:disable Style/GuardClause, Lint/NonLocalExitFromIterator
         dependent_queues.each do |qname|
           latency = Amigo::QueueBackoffJob.check_latency(qname)

data/lib/amigo/semaphore_backoff_job.rb

@@ -2,9 +2,8 @@
 
 require "sidekiq"
 
-module Amigo
-  # This is a placeholder until it's migrated to Amigo proper
-end
+require "amigo"
+require "amigo/memory_pressure"
 
 # Semaphore backoff jobs can reschedule themselves to happen at a later time
 # if there is too high a contention on a semaphore.
@@ -48,6 +47,11 @@ end
 # around missing decrements), if we ever detect a negative 'jobs running',
 # we warn and remove the key entirely.
 #
+# == High Memory Utilization
+#
+# Queue backoff behavior is automatically disabled under high memory utilization,
+# as per +Amigo::MemoryPressure+.
+#
 module Amigo
   module SemaphoreBackoffJob
     def self.included(cls)
@@ -94,6 +98,7 @@ module Amigo
       def perform(*args)
         self.before_perform(*args) if self.respond_to?(:before_perform)
         return super unless ::Amigo::SemaphoreBackoffJob.enabled?
+        return super if ::Amigo::MemoryPressure.instance.under_pressure?
         key = self.semaphore_key
         size = self.semaphore_size
         # Create a simple counter for the semaphore key.

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Amigo
-  VERSION = "1.8.0"
+  VERSION = "1.10.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.10.0
 platform: ruby
 authors:
 - Lithic Technology
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-04-27 00:00:00.000000000 Z
+date: 2025-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -179,6 +179,7 @@ files:
 - lib/amigo/autoscaler/heroku.rb
 - lib/amigo/deprecated_jobs.rb
 - lib/amigo/job.rb
+- lib/amigo/memory_pressure.rb
 - lib/amigo/queue_backoff_job.rb
 - lib/amigo/rate_limited_error_handler.rb
 - lib/amigo/retry.rb