sidekiq-amigo 1.9.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 330ead7f8face652362d632c9c127aa57b298d5587d1da392769b0458d1f8915
-  data.tar.gz: 7c0b0b9b1658eecfd202ef294580417233421edf4233eef522b747ce9c411f52
+  metadata.gz: 5e1aa001f6060b1e13decb95357627151007e8f17066cef3401021103a3cd7a2
+  data.tar.gz: 955cb548d0739f5fa51bc0fe887a4e0df5d15c154ee5470e4bdba4883a308af4
 SHA512:
-  metadata.gz: 898c24b82de6313c33e50b58a2c4d64ca365918a0aee2b223b452fea9428cc4d265b15975ebc2f7c7e689f547898e65b7c52d3e50edbe9bcdb062f3b25f3f0b6
-  data.tar.gz: a5ffcb482859fcddd6727a9869bef3ed00c8edb5b1b44f0cde84b31eddb44c6f84fbac6b45d8b8fccf06c712f6792a8ce3e211b4a49083fe8c0bab76474a5d38
+  metadata.gz: 9fb081204d9465257ac48df3193925bb547961971fc0db39ea235584e9f4b9acef7ab2f93eaed60c1dcd4d1de759d09daadc5a94ec89473e600eb72b67c4afc7
+  data.tar.gz: bea97ecddf27912029035a941e6ee3aff653c0f8ba958e27f21a539f984a987c017311aa21b5f306832bb2240a0f487522e0cd63bf5a87988e5e568ebf503c7b

data/lib/amigo/audit_logger.rb

@@ -4,7 +4,7 @@ require "amigo"
 
 module Amigo
   class AuditLogger
-    include Sidekiq::Worker
+    include Sidekiq::Job
 
     def audit_log_level
       return :info

data/lib/amigo/autoscaler.rb

@@ -35,6 +35,13 @@ module Amigo
   class Autoscaler
     class InvalidHandler < StandardError; end
 
+    # Struct representing data serialized to Redis.
+    # Useful for diagnostics. Can be retried with +fetch_persisted+.
+    # @!attribute last_alerted_at [Time] 0-time if there is no recent alert.
+    # @!attribute depth [Integer] 0 if not in a latency event.
+    # @!attribute latency_event_started_at [Time] 0-time if not in a latency event.
+    Persisted = Struct.new(:last_alerted_at, :depth, :latency_event_started_at)
+
     # How often should Autoscaler check for latency?
     # @return [Integer]
     attr_reader :poll_interval
@@ -139,10 +146,19 @@ module Amigo
       @alert_methods = self.handlers.map { |a| _handler_to_method("alert_", a) }
       @restored_methods = self.latency_restored_handlers.map { |a| _handler_to_method("alert_restored_", a) }
       @stop = false
-      Sidekiq.redis do |r|
-        @last_alerted = Time.at((r.get("#{namespace}/last_alerted") || 0).to_f)
-        @depth = (r.get("#{namespace}/depth") || 0).to_i
-        @latency_event_started = Time.at((r.get("#{namespace}/latency_event_started") || 0).to_f)
+      persisted = self.fetch_persisted
+      @last_alerted = persisted.last_alerted_at
+      @depth = persisted.depth
+      @latency_event_started = persisted.latency_event_started_at
+    end
+
+    def fetch_persisted
+      return Sidekiq.redis do |r|
+        Persisted.new(
+          Time.at((r.get("#{namespace}/last_alerted") || 0).to_f),
+          (r.get("#{namespace}/depth") || 0).to_i,
+          Time.at((r.get("#{namespace}/latency_event_started") || 0).to_f),
+        )
       end
     end
 

data/lib/amigo/job.rb

@@ -7,7 +7,7 @@ require "amigo"
 module Amigo
   module Job
     def self.extended(cls)
-      cls.include(Sidekiq::Worker)
+      cls.include(Sidekiq::Job)
       cls.extend(ClassMethods)
       cls.pattern = ""
       cls.include(InstanceMethods)

data/lib/amigo/memory_pressure.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Amigo
+  # Helper class to detect when the Redis server is under memory pressure.
+  # In these cases, we want to disable queue backoff behavior.
+  # There is a significant risk that the backoff behavior will take jobs from the queue,
+  # and immediately try and reschedule them. If that happens in an OOM condition,
+  # the re-push will fail and the job can be lost.
+  #
+  # Additionally, the backoff behavior causes delays that slow down the clearing of the queue.
+  #
+  # In these high-memory-utilization conditions, it makes more sense to disable the backoff logic
+  # and just brute force to try to get through the queue.
+  class MemoryPressure
+    # Percentage at which the server is considered under memory pressure.
+    DEFAULT_THRESHOLD = 90
+
+    # Default seconds a memory check is good for. See +check_ttl+.
+    DEFAULT_CHECK_TTL = 120
+
+    class << self
+      # Return the singleton instance, creating a cached value if needed.
+      def instance
+        return @instance ||= self.new
+      end
+
+      # Set the instance, or use nil to reset.
+      attr_writer :instance
+    end
+
+    # When did we last check for pressure?
+    attr_reader :last_checked_at
+
+    # What was the result of the last check?
+    # true is under pressure, false if not.
+    attr_reader :last_check_result
+
+    # See +DEFAULT_CHECK_TTL+.
+    attr_reader :check_ttl
+
+    # See +DEFAULT_THRESHOLD+.
+    attr_reader :threshold
+
+    def initialize(check_ttl: DEFAULT_CHECK_TTL, threshold: DEFAULT_THRESHOLD)
+      @last_checked_at = nil
+      @check_ttl = check_ttl
+      @threshold = threshold
+      @last_check_result = nil
+    end
+
+    # Return true if the server is under memory pressure.
+    # When this is the case, we want to disable backoff,
+    # since it will delay working through the queue,
+    # and can also result in a higher likelihood of lost jobs,
+    # since returning them back to the queue will fail.
+    def under_pressure?
+      return @last_check_result unless self.needs_check?
+      @last_check_result = self.calculate_under_pressure
+      @last_checked_at = Time.now
+      return @last_check_result
+    end
+
+    private def needs_check?
+      return true if @last_checked_at.nil?
+      return (@last_checked_at + @check_ttl) < Time.now
+    end
+
+    private def calculate_under_pressure
+      meminfo = self.get_memory_info
+      used_bytes = meminfo.fetch("used_memory", "0").to_f
+      max_bytes = meminfo.fetch("maxmemory", "0").to_f
+      return false if used_bytes.zero? || max_bytes.zero?
+      percentage = (used_bytes / max_bytes) * 100
+      return percentage > self.threshold
+    end
+
+    def get_memory_info
+      s = self.get_memory_info_string
+      return self.parse_memory_string(s)
+    end
+
+    protected def get_memory_info_string
+      s = Sidekiq.redis do |c|
+        c.call("INFO", "MEMORY")
+      end
+      return s
+    end
+
+    protected def parse_memory_string(s)
+      # See bottom of https://redis.io/docs/latest/commands/info/ for format.
+      pairs = s.split("\r\n").reject { |line| line.start_with?("#") }.map { |pair| pair.split(":", 2) }
+      h = pairs.to_h
+      return h
+    end
+  end
+end

data/lib/amigo/queue_backoff_job.rb

@@ -3,6 +3,8 @@
 require "sidekiq"
 require "sidekiq/api"
 
+require "amigo/memory_pressure"
+
 # Queue backoff jobs are used for jobs that should not saturate workers,
 # such that jobs on dependent queues end up not running for a while.
 #
@@ -45,6 +47,11 @@ require "sidekiq/api"
 # This is a fast call (it just gets the last item), but it's not free,
 # so users should be aware of it.
 #
+# == High Memory Utilization
+#
+# Queue backoff behavior is automatically disabled under high memory utilization,
+# as per +Amigo::MemoryPressure+.
+#
 module Amigo
   module QueueBackoffJob
     def self.included(cls)
@@ -126,6 +133,7 @@ module Amigo
     module PrependedMethods
       def perform(*args)
         return super unless ::Amigo::QueueBackoffJob.enabled?
+        return super if ::Amigo::MemoryPressure.instance.under_pressure?
         # rubocop:disable Style/GuardClause, Lint/NonLocalExitFromIterator
         dependent_queues.each do |qname|
           latency = Amigo::QueueBackoffJob.check_latency(qname)

data/lib/amigo/retry.rb

@@ -3,7 +3,7 @@
 require "sidekiq"
 require "sidekiq/api"
 
-# Middleware so Sidekiq workers can use a custom retry logic.
+# Middleware so Sidekiq jobs can use a custom retry logic.
 # See +Amigo::Retry::Retry+, +Amigo::Retry::Die+,
 # and +Amigo::Retry::OrDie+ for more details
 # on how these should be used.
@@ -83,6 +83,8 @@ module Amigo
     end
 
     class ServerMiddleware
+      include Sidekiq::ServerMiddleware
+
       def call(worker, job, _queue)
         yield
       rescue Amigo::Retry::Retry => e
@@ -120,14 +122,14 @@ module Amigo
         end
       end
 
-      def amigo_retry_in(worker_class, item, interval)
+      def amigo_retry_in(job_class, item, interval)
         # pulled from perform_in
         int = interval.to_f
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
         item["at"] = ts if ts > now
         item["retry_count"] = item.fetch("retry_count", 0) + 1
-        worker_class.client_push(item)
+        job_class.client_push(item)
       end
     end
   end

data/lib/amigo/router.rb

@@ -6,7 +6,7 @@ require "amigo"
 
 module Amigo
   class Router
-    include Sidekiq::Worker
+    include Sidekiq::Job
 
     def perform(event_json)
       event_name = event_json["name"]

data/lib/amigo/scheduled_job.rb

@@ -8,7 +8,7 @@ require "amigo"
 module Amigo
   module ScheduledJob
     def self.extended(cls)
-      cls.include(Sidekiq::Worker)
+      cls.include(Sidekiq::Job)
       cls.sidekiq_options(retry: false)
       cls.extend(ClassMethods)
       cls.splay_duration = 30

data/lib/amigo/semaphore_backoff_job.rb

@@ -2,9 +2,8 @@
 
 require "sidekiq"
 
-module Amigo
-  # This is a placeholder until it's migrated to Amigo proper
-end
+require "amigo"
+require "amigo/memory_pressure"
 
 # Semaphore backoff jobs can reschedule themselves to happen at a later time
 # if there is too high a contention on a semaphore.
@@ -34,7 +33,7 @@ end
 # - `semaphore_expiry` should return the TTL of the semaphore key.
 #   Defaults to 30 seconds. See below for key expiry and negative semaphore value details.
 # - `before_perform` is called before calling the `perform` method.
-#   This is required so that implementers can set worker state, based on job arguments,
+#   This is required so that implementers can set job state, based on job arguments,
 #   that can be used for calculating the semaphore key.
 #
 # Note that we give the semaphore key an expiry. This is to avoid situation where
@@ -42,12 +41,17 @@ end
 # have fewer than the expected number of jobs running.
 #
 # This does mean that, when a job runs longer than the semaphore expiry,
-# another worker can be started, which would increment the counter back to 1.
+# another job can be started, which would increment the counter back to 1.
 # When the original job ends, the counter would be 0; then when the new job ends,
 # the counter would be -1. To avoid negative counters (which create the same issue
 # around missing decrements), if we ever detect a negative 'jobs running',
 # we warn and remove the key entirely.
 #
+# == High Memory Utilization
+#
+# Queue backoff behavior is automatically disabled under high memory utilization,
+# as per +Amigo::MemoryPressure+.
+#
 module Amigo
   module SemaphoreBackoffJob
     def self.included(cls)
@@ -74,11 +78,11 @@ module Amigo
 
     module InstanceMethods
       def semaphore_key
-        raise NotImplementedError, "must be implemented on worker"
+        raise NotImplementedError, "must be implemented on job"
       end
 
       def semaphore_size
-        raise NotImplementedError, "must be implemented on worker"
+        raise NotImplementedError, "must be implemented on job"
       end
 
       def semaphore_backoff
@@ -94,6 +98,7 @@ module Amigo
       def perform(*args)
         self.before_perform(*args) if self.respond_to?(:before_perform)
         return super unless ::Amigo::SemaphoreBackoffJob.enabled?
+        return super if ::Amigo::MemoryPressure.instance.under_pressure?
         key = self.semaphore_key
         size = self.semaphore_size
         # Create a simple counter for the semaphore key.

data/lib/amigo/spec_helpers.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "amigo"
-require "sidekiq/worker"
 
 module Amigo
   module SpecHelpers
@@ -248,7 +247,7 @@ module Amigo
       return PerformAsyncJobMatcher.new(job)
     end
 
-    # Like a Sidekiq worker's perform_inline,
+    # Like a Sidekiq job's perform_inline,
     # but allows an arbitrary item to be used, rather than just the
     # given class and args. For example, when testing,
     # you may need to assume something like 'retry_count' is in the job payload,
@@ -256,18 +255,18 @@ module Amigo
     # This allows those arbitrary job payload fields
     # to be included when the job is run.
     module_function def sidekiq_perform_inline(klass, args, item=nil)
-      Sidekiq::Worker::Setter.override_item = item
+      Sidekiq::Job::Setter.override_item = item
       begin
         klass.perform_inline(*args)
       ensure
-        Sidekiq::Worker::Setter.override_item = nil
+        Sidekiq::Job::Setter.override_item = nil
       end
     end
 
     module_function def drain_sidekiq_jobs(q)
       all_sidekiq_jobs(q).each do |job|
         klass = job.item.fetch("class")
-        klass = Sidekiq::Testing.constantize(klass) if klass.is_a?(String)
+        klass = Object.const_get(klass) if klass.is_a?(String)
         sidekiq_perform_inline(klass, job.item["args"], job.item)
         job.delete
       end
@@ -282,6 +281,8 @@ module Amigo
     # Use this middleware to pass an arbitrary callback evaluated before a job runs.
     # Make sure to call +reset+ after the test.
     class ServerCallbackMiddleware
+      include Sidekiq::ServerMiddleware
+
       class << self
         attr_accessor :callback
       end
@@ -304,7 +305,7 @@ module Amigo
 end
 
 module ::Sidekiq
-  module Worker
+  module Job
     class Setter
       class << self
         attr_accessor :override_item

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Amigo
-  VERSION = "1.9.0"
+  VERSION = "1.11.0"
 end

data/lib/amigo.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "redis"
 require "sidekiq"
 require "sidekiq-cron"
 
@@ -61,18 +60,18 @@ require "sidekiq-cron"
 # to control the matching rules more closely than File.fnmatch can provide.
 #
 # Jobs must implement a `_perform` method, which takes a Amigo::Event.
-# Note that normal Sidekiq workers use a 'perform' method that takes a variable number of arguments;
+# Note that normal Sidekiq jobs use a 'perform' method that takes a variable number of arguments;
 # the base Async::Job class has this method and delegates its business logic to the subclass _perform method.
 #
 # Routing
 #
-# There are two special workers that are important for the overall functioning of the system
-# (and do not inherit from Job but rather than Sidekiq::Worker so they are not classified and treated as 'Jobs').
+# There are two special jobs that are important for the overall functioning of the system
+# (and do not inherit from Job but rather than Sidekiq::Job so they are not classified and treated as 'Jobs').
 #
 # The first is the AuditLogger, which is a basic job that logs all async events.
 # This acts as a useful change log for the state of the database.
 #
-# The second special worker is the Router, which calls `perform` on the event Jobs
+# The second special job is the Router, which calls `perform` on the event Jobs
 # that match the routing information, as explained in Jobs.
 # It does this by filtering through all event-based jobs and performing the ones with a route match.
 #

metadata

@@ -1,43 +1,42 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Lithic Technology
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-02-03 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: sidekiq-cron
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: platform-api
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +57,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -179,6 +178,7 @@ files:
 - lib/amigo/autoscaler/heroku.rb
 - lib/amigo/deprecated_jobs.rb
 - lib/amigo/job.rb
+- lib/amigo/memory_pressure.rb
 - lib/amigo/queue_backoff_job.rb
 - lib/amigo/rate_limited_error_handler.rb
 - lib/amigo/retry.rb
@@ -192,7 +192,6 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -200,15 +199,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: 3.2.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Pubsub system and other enhancements around Sidekiq.
 test_files: []