aeternitas 0.2.0 → 2.0.0.rc1

data/lib/aeternitas/guard.rb

@@ -1,9 +1,9 @@
-require 'active_support/duration'
-require 'securerandom'
+require "active_support/duration"
+require "securerandom"
 
 module Aeternitas
   # A distributed lock that can not be acquired after being unlocked for a certain time (cooldown period).
-  # Using Redis key expiration we ensure locks are released even after workers crash after a configurable timout period.
+  # Using a database table (`aeternitas_guard_locks`) with pessimistic locking we ensure atomicity and prevent race conditions.
   #
   # @example
   #   guard = Aeternitas::Guard.new("Twitter-MY_API_KEY", 5.seconds)
@@ -13,7 +13,7 @@ module Aeternitas
   #     end
   #   rescue Twitter::TooManyRequests => e
   #     guard.sleep_until(e.rate_limit.reset_at)
-  #     raise Aeternitas::Guard::GuardIsLocked(e.rate_limit.reset_at)
+  #     raise Aeternitas::Guard::GuardIsLocked.new(e.rate_limit.reset_at)
   #   end
   #
   # @!attribute [r] id
@@ -25,7 +25,6 @@ module Aeternitas
   # @!attribute [r] token
   #   @return [String] cryptographic token which ensures we do not lock/unlock a guard held by another process
   class Guard
-
     attr_reader :id, :timeout, :cooldown, :token
 
     # Create a new Guard
@@ -35,15 +34,15 @@ module Aeternitas
     # @param [ActiveRecord::Duration] timeout Lock timeout
     # @return [Aeternitas::Guard] Creates a new Instance
     def initialize(id, cooldown, timeout = 10.minutes)
-      @id       = id
+      @id = id
       @cooldown = cooldown
-      @timeout  = timeout
-      @token    = SecureRandom.hex(10)
+      @timeout = timeout
+      @token = SecureRandom.hex(10)
     end
 
     # Runs a given block if the lock can be acquired and releases the lock afterwards.
     #
-    # @raise [Aeternitas::LockWithCooldown::GuardIsLocked] if the lock can not be acquired
+    # @raise [Aeternitas::Guard::GuardIsLocked] if the lock can not be acquired
     # @example
     #   Guard.new("MyId", 5.seconds, 10.minutes).with_lock { do_request() }
     def with_lock
@@ -55,6 +54,22 @@ module Aeternitas
       end
     end
 
+    # Tries to unlock the guard and starts the cooldown phase.
+    # It only releases the lock if the token matches and the state is 'processing'.
+    def unlock
+      Aeternitas::GuardLock.transaction do
+        lock = Aeternitas::GuardLock.where(lock_key: @id, token: @token).lock.first
+        return false unless lock&.processing?
+
+        lock.update!(
+          state: :cooldown,
+          locked_until: @cooldown.from_now,
+          reason: nil
+        )
+      end
+      true
+    end
+
     # Locks the guard until the given time.
     #
     # @param [Time] until_time sleep time
@@ -68,118 +83,69 @@ module Aeternitas
     # @param [ActiveSupport::Duration] duration sleeping duration
     # @param [String] msg hint why the guard sleeps
     def sleep_for(duration, msg = nil)
-      raise ArgumentError, 'duration must be an ActiveRecord::Duration' unless duration.is_a?(ActiveSupport::Duration)
+      raise ArgumentError, "duration must be an ActiveRecord::Duration" unless duration.is_a?(ActiveSupport::Duration)
       sleep_until(duration.from_now, msg)
     end
 
     private
 
     # Tries to acquire the lock.
-    #
-    # @example The Redis value looks like this
-    #   {
-    #     id: 'MyId'
-    #     state: 'processing'
-    #     timeout: '3600'
-    #     cooldown: '5'
-    #     locked_until: '2017-01-01 10:10:00'
-    #     token: '1234567890'
-    #   }
     # @raise [Aeternitas::Guard::GuardIsLocked] if the lock can not be acquired
     def acquire_lock!
-      payload = {
-        'id' => @id,
-        'state' => 'processing',
-        'timeout' => @timeout,
-        'cooldown' => @cooldown,
-        'locked_until' => @timeout.from_now,
-        'token' => @token
-      }
-
-      has_lock = Aeternitas.redis.set(@id, JSON.unparse(payload), ex: @timeout.to_i, nx: true)
-
-      raise(GuardIsLocked.new(@id, get_timeout)) unless has_lock
-    end
-
-    # Tries to unlock the guard. This starts the cooldown phase.
-    #
-    # @example The Redis value looks like this
-    #   {
-    #     id: 'MyId'
-    #     state: 'cooldown'
-    #     timeout: '3600'
-    #     cooldown: '5'
-    #     locked_until: '2017-01-01 10:00:05'
-    #     token: '1234567890'
-    #   }
-    def unlock
-      return false unless holds_lock?
-
-      payload = {
-          'id' => @id,
-          'state' => 'cooldown',
-          'timeout' => @timeout,
-          'cooldown' => @cooldown,
-          'locked_until' => @cooldown.from_now,
-          'token' => @token
-      }
-
-      Aeternitas.redis.set(@id, JSON.unparse(payload), ex: @cooldown.to_i)
+      retries = 0
+      begin
+        Aeternitas::GuardLock.transaction do
+          lock = Aeternitas::GuardLock.where(lock_key: @id).lock.first
+
+          if lock
+            if lock.locked_until > Time.current
+              # Lock is still active
+              raise GuardIsLocked.new(@id, lock.locked_until, lock.reason)
+            else
+              # Lock has expired
+              lock.update!(
+                token: @token,
+                state: :processing,
+                locked_until: @timeout.from_now,
+                reason: nil
+              )
+            end
+          else
+            # Create new lock
+            Aeternitas::GuardLock.create!(
+              lock_key: @id,
+              token: @token,
+              state: :processing,
+              locked_until: @timeout.from_now
+            )
+          end
+        end
+      rescue ActiveRecord::RecordNotUnique
+        # prevent infinite loops in unexpected scenarios
+        retries += 1
+        raise if retries > 4
+        retry
+      end
     end
 
-    # Lets the guard sleep until the given date.
-    # This means that non can acquire the guards lock
-    #
-    # @example The Redis value looks like this
-    #   {
-    #     id: 'MyId'
-    #     state: 'sleeping'
-    #     timeout: '3600'
-    #     cooldown: '5'
-    #     locked_until: '2017-01-01 13:00'
-    #     message: "API Quota Reached"
-    #   }
+    # Lets the guard sleep until the given time.
+    # This will create a new sleeping lock or update an existing one.
     # @todo Should this raise an error if the lock is not owned by this instance?
     # @param [Time] sleep_timeout for how long will the guard sleep
     # @param [String] msg hint why the guard sleeps
     def sleep(sleep_timeout, msg = nil)
-      payload = {
-          'id' => @id,
-          'state' => 'sleeping',
-          'timeout' => @timeout,
-          'cooldown' => @cooldown,
-          'locked_until' => sleep_timeout
-      }
-      payload.merge(message: msg) if msg
+      Aeternitas::GuardLock.transaction do
+        lock = Aeternitas::GuardLock.where(lock_key: @id).lock.first_or_initialize
 
-      Aeternitas.redis.set(@id, JSON.unparse(payload), ex: (sleep_timeout - Time.now).seconds.to_i)
-    end
+        lock.assign_attributes(
+          token: @token,
+          state: :sleeping,
+          locked_until: sleep_timeout,
+          reason: msg
+        )
 
-    # Checks if this instance holds the lock. This is done by retrieving the value from redis and
-    # comparing the token value. If they match, than the lock is held by this instance.
-    #
-    # @todo Make the check atomic
-    # @return [Boolean] if the lock is held by this instance
-    def holds_lock?
-      payload = get_payload
-      payload['token'] == @token && payload['state'] == 'processing'
-    end
-
-    # Returns the guards current timeout.
-    #
-    # @return [Time] the guards current timeout
-    def get_timeout
-      payload = get_payload
-      payload['state'] == 'processing' ? payload['cooldown'].to_i.seconds.from_now : Time.parse(payload['locked_until'])
-    end
-
-    # Retrieves the locks payload from redis.
-    #
-    # @return [Hash] the locks payload
-    def get_payload
-      value = Aeternitas.redis.get(@id)
-      return {} unless value
-      JSON.parse(value)
+        lock.save!
+      end
     end
 
     # Custom error class thrown when the lock can not be acquired
@@ -196,4 +162,4 @@ module Aeternitas
       end
     end
   end
-end
+end

data/lib/aeternitas/guard_lock.rb

@@ -0,0 +1,19 @@
+require "active_record"
+
+module Aeternitas
+  # Stores the state of a distributed lock in the database.
+  class GuardLock < ActiveRecord::Base
+    self.table_name = "aeternitas_guard_locks"
+
+    enum :state, {
+      processing: "processing",
+      cooldown: "cooldown",
+      sleeping: "sleeping"
+    }
+
+    validates :lock_key, presence: true, uniqueness: true
+    validates :locked_until, presence: true
+    validates :token, presence: true
+    validates :state, presence: true, inclusion: {in: states.keys}
+  end
+end

data/lib/aeternitas/maintenance.rb

@@ -0,0 +1,35 @@
+module Aeternitas
+  # Provides methods for cleaning up Aeternitas data.
+  module Maintenance
+    def self.cleanup_all
+      cleanup_stale_locks
+      cleanup_old_metrics
+    end
+
+    # Clean up stale job and guard locks that have passed their expiration.
+    def self.cleanup_stale_locks
+      logger = ActiveJob::Base.logger
+      logger.info "Cleaning up stale Aeternitas locks..."
+
+      unique_job_locks_deleted = Aeternitas::UniqueJobLock.where("expires_at < ?", Time.current).delete_all
+      guard_locks_deleted = Aeternitas::GuardLock.where("locked_until < ?", Time.current).delete_all
+
+      logger.info "  - Deleted #{unique_job_locks_deleted} stale unique job locks."
+      logger.info "  - Deleted #{guard_locks_deleted} stale guard locks."
+      logger.info "Stale lock cleanup complete."
+    end
+
+    # Clean up old metric records to prevent the table from growing too large.
+    def self.cleanup_old_metrics
+      logger = ActiveJob::Base.logger
+      cutoff_date = Aeternitas.config.metric_retention_period.ago
+
+      logger.info "Cleaning up Aeternitas metrics older than #{cutoff_date}..."
+
+      metrics_deleted = Aeternitas::Metric.where("created_at < ?", cutoff_date).delete_all
+
+      logger.info "  - Deleted #{metrics_deleted} old metric records."
+      logger.info "Old metrics cleanup complete."
+    end
+  end
+end

data/lib/aeternitas/metric.rb

@@ -0,0 +1,12 @@
+require "active_record"
+
+module Aeternitas
+  # Stores a single metric data point in the database.
+  class Metric < ActiveRecord::Base
+    self.table_name = "aeternitas_metrics"
+
+    validates :name, presence: true
+    validates :pollable_class, presence: true
+    validates :value, presence: true
+  end
+end

data/lib/aeternitas/metrics.rb

@@ -1,18 +1,14 @@
-require 'aeternitas/metrics/ten_minutes_resolution'
-require 'aeternitas/metrics/counter'
-require 'aeternitas/metrics/values'
-require 'aeternitas/metrics/ratio'
-
 module Aeternitas
-  # Provides extensive metrics for Aeternitas.
-  # Every metric is scoped by pollable class.
+  # Provides a simplified metrics system for Aeternitas.
+  # Every metric is scoped by pollable class and logged in the `aeternitas_metrics` table.
+  #
   # Available metrics are:
   #   - polls => Number of polling runs
   #   - successful_polls => Number of successful polling runs
   #   - failed_polls => Number of failed polling runs (includes IgnoredErrors,
   #     excludes deactivation errors and Lock errors)
   #   - ignored_errors => Number of raised {Aeternitas::Errors::Ignored}
-  #   - deactivation_errors => Number of errors raised which are declared as deactivation_errors
+  #   - deactivations => Number of deactivations
   #   - execution_time => Job execution time in seconds
   #   - guard_locked => Number of encountered locked guards
   #   - guard_timeout => Time until the guard is unlocked in seconds
@@ -20,143 +16,65 @@ module Aeternitas
   #   - pollables_created => Number of created pollables
   #   - sources_created => Number of created sources
   #
-  # Available Resolutions are:
-  #   - :minute (stored for 3 days)
-  #   - :ten_minutes (stored for 14 days)
-  #   - :hour (stored for 2 months)
-  #   - :day (stored indefinitely)
-  #
-  # Every metric can be accessed via a getter method:
   # @example
-  #   Aeternitas::Metrics.polls MyPollable, from: 3.days.ago, to: Time.now, resolution: :hour
-  #   Aeternitas::Metrics.execution_times MyPollable
-  # @see #get
+  #   Aeternitas::Metrics.log(:polls, MyPollable)
+  #   Aeternitas::Metrics.log_value(:execution_time, MyPollable, 1.25)
+  #
+  #   # Get all poll counts for MyPollable in the last day
+  #   Aeternitas::Metrics.get(:polls, MyPollable, from: 1.day.ago)
+  #
   module Metrics
-    AVAILABLE_METRICS = {
-      polls: :counter,
-      successful_polls: :counter,
-      failed_polls: :counter,
-      ignored_errors: :counter,
-      deactivations: :counter,
-      execution_time: :value,
-      guard_locked: :counter,
-      guard_timeout: :value,
-      guard_timeout_exceeded: :counter,
-      sources_created: :counter,
-      pollables_created: :counter
-    }.freeze
-
-    TabsTabs.configure do |tabstabs_config|
-      tabstabs_config.unregister_resolutions(:week, :month, :year)
-
-      tabstabs_config.register_resolution Aeternitas::Metrics::TenMinutesResolution
-
-      tabstabs_config.set_expirations(
-        minute: 3.days,
-        ten_minutes: 14.days,
-        hour: 2.months
-      )
-    end
+    # A list of all available metric names.
+    AVAILABLE_METRICS = [
+      :polls,
+      :successful_polls,
+      :failed_polls,
+      :ignored_errors,
+      :deactivations,
+      :execution_time,
+      :guard_locked,
+      :guard_timeout,
+      :guard_timeout_exceeded,
+      :sources_created,
+      :pollables_created
+    ].freeze
 
-    AVAILABLE_METRICS.each_pair do |metric, _|
-      module_eval <<-eoruby, __FILE__, __LINE__ + 1
-        def self.#{metric}(pollable, from: 1.hour.ago, to: Time.now, resolution: :minute)
-          self.get(:#{metric}, pollable, from: from, to: to, resolution: resolution )
-        end
-      eoruby
-    end
-
-    # Increses the specified counter metric for the given pollable.
-    # @param [Symbol, String] name the metric
-    # @param [Pollable] pollable_class pollable instance
+    # Logs a counter metric. This creates a new metric record with a value of 1.
+    # @param [Symbol] name the metric name
+    # @param [Class] pollable_class the class of the pollable
     def self.log(name, pollable_class)
-      raise('Metric not found') unless AVAILABLE_METRICS.key? name
-      raise ArgumentError, "#{name} isn't a Counter" unless AVAILABLE_METRICS[name] == :counter
-      begin
-        TabsTabs.increment_counter(get_key(name, pollable_class))
-        TabsTabs.increment_counter(get_key(name, Aeternitas::Pollable))
-      rescue StandardError ; end
+      log_value(name, pollable_class, 1)
     end
 
-    # Logs a value in a value metric for the given pollable.
-    # @param [Symbol String] name the metric
-    # @param [Pollable] pollable_class pollable instance
-    # @param [Object] value the value
+    # Logs a value-based metric.
+    # @param [Symbol] name the metric name
+    # @param [Class] pollable_class the class of the pollable
+    # @param [Float] value the value to log
     def self.log_value(name, pollable_class, value)
-      raise('Metric not found') unless AVAILABLE_METRICS.key? name
-      raise(ArgumentError, "#{name} isn't a Value") unless AVAILABLE_METRICS[name] == :value
-      begin
-        TabsTabs.record_value(get_key(name, pollable_class), value)
-        TabsTabs.record_value(get_key(name, Aeternitas::Pollable), value)
-      rescue StandardError ; end
-    end
-
-    # Retrieves the stats of the given metric in the given time frame and resolution.
-    # @param [Symbol String] name the metric
-    # @param [Object] pollable_class the pollable class
-    # @param [DateTime] from begin of the time frame
-    # @param [DateTime] to end of the timeframe
-    # @param [Symbol] resolution resolution
-    # @return [Aeternitas::Metrics::Counter, Aeternitas::Metrics::Value] stats
-    def self.get(name, pollable_class, from: 1.hour.ago, to: Time.now, resolution: :minute)
-      raise('Metric not found') unless AVAILABLE_METRICS.key? name
-      raise('Invalid interval') if from > to
-      result = TabsTabs.get_stats(get_key(name, pollable_class), from..to, resolution)
-      if AVAILABLE_METRICS[name] == :counter
-        Counter.new(result)
-      else
-        Values.new(result)
-      end
-    rescue TabsTabs::UnknownMetricError => _
-      TabsTabs.create_metric(get_key(name, pollable_class), AVAILABLE_METRICS[name].to_s)
-      get(name, pollable_class, from: from, to: to, resolution: resolution)
-    end
+      return unless Aeternitas.config.metrics_enabled && AVAILABLE_METRICS.include?(name)
 
-    # Returns the failure ratio of the given job for given time frame and resolution
-    # @param [Symbol String] name the metric
-    # @param [Object] pollable_class the pollable class
-    # @param [DateTime] from begin of the time frame
-    # @param [DateTime] to end of the timeframe
-    # @param [Symbol] resolution resolution
-    # @return [Aeternitas::Metrics::Ratio] ratio time series
-    def self.failure_ratio(pollable_class, from: 1.hour.ago, to: Time.now, resolution: :minute)
-      polls = polls(pollable_class, from: from, to: to, resolution: resolution)
-      failed_polls = failed_polls(pollable_class, from: from, to: to, resolution: resolution)
-      Ratio.new(from, to, resolution, calculate_ratio(polls, failed_polls))
-    end
-
-    # Returns the lock ratio of the given job for given time frame and resolution
-    # @param [Symbol String] name the metric
-    # @param [Object] pollable_class the pollable class
-    # @param [DateTime] from begin of the time frame
-    # @param [DateTime] to end of the timeframe
-    # @param [Symbol] resolution resolution
-    # @return [Aeternitas::Metrics::Ratio] ratio time series
-    def self.guard_locked_ratio(pollable_class, from: 1.hour.ago, to: Time.now, resolution: :minute)
-      polls = polls(pollable_class, from: from, to: to, resolution: resolution)
-      guard_locked = guard_locked(pollable_class, from: from, to: to, resolution: resolution)
-      Ratio.new(from, to, resolution, calculate_ratio(polls, guard_locked))
-    end
-
-    # Computes the metric key of a given metric-pollable pair
-    # @param [Symbol, String] name the metric
-    # @param [Object] pollable_class pollable class
-    # @return [String] the metric key
-    def self.get_key(name, pollable_class)
-      "#{name}:#{pollable_class.name}"
+      Aeternitas::Metric.create(
+        name: name.to_s,
+        pollable_class: pollable_class.name,
+        value: value,
+        created_at: Time.now
+      )
+    rescue
+      # Metrics should fail silently
     end
 
-    # Computes the ratio of a base counter time series and a target counter time series
-    # @param [Array] base base time series data
-    # @param [Array] target target time series data
-    # @return [Array] ratio time series data
-    def self.calculate_ratio(base, target)
-      base.zip(target).map do |b, t|
-        {
-          timestamp: b['timestamp'],
-          ratio: b['count'].to_i.zero? ? 0 : t['count'].to_i / b['count'].to_f
-        }.with_indifferent_access
-      end
+    # Retrieves metric records.
+    # @param [Symbol] name the metric
+    # @param [Class] pollable_class the pollable class
+    # @param [Time] from begin of the time frame
+    # @param [Time] to end of the timeframe
+    # @return [ActiveRecord::Relation] a relation of Aeternitas::Metric records
+    def self.get(name, pollable_class, from: 1.hour.ago, to: Time.now)
+      Aeternitas::Metric.where(
+        name: name.to_s,
+        pollable_class: pollable_class.name,
+        created_at: from..to
+      ).order(:created_at)
     end
   end
-end
+end

data/lib/aeternitas/poll_job.rb

@@ -0,0 +1,135 @@
+require_relative "aeternitas_job"
+require "digest"
+
+module Aeternitas
+  # ActiveJob worker responsible for executing the polling.
+  class PollJob < AeternitasJob
+    queue_as :polling
+
+    LOCK_EXPIRATION = 1.month
+    RETRY_DELAYS = [60.seconds, 1.hour, 1.day, 1.week].freeze
+    MAX_TOTAL_ATTEMPTS = 5
+
+    # === Job Uniqueness ===
+    before_enqueue do |job|
+      # Only check for uniqueness on the first attempt. Retries should not be blocked by their own lock.
+      if job.executions.zero?
+        pollable_meta_data_id = job.arguments.first
+        pollable_meta_data = Aeternitas::PollableMetaData.find(pollable_meta_data_id)
+        pollable = pollable_meta_data.pollable
+
+        lock_digest = self.class.generate_lock_digest(pollable_meta_data_id)
+        guard_key_digest = self.class.generate_guard_key_digest(pollable)
+
+        Aeternitas::UniqueJobLock.where("lock_digest = ? AND expires_at <= ?", lock_digest, Time.now).destroy_all
+
+        new_lock = Aeternitas::UniqueJobLock.new(
+          lock_digest: lock_digest,
+          guard_key_digest: guard_key_digest,
+          expires_at: Time.now + LOCK_EXPIRATION,
+          job_id: job.job_id
+        )
+
+        unless new_lock.save
+          ActiveJob::Base.logger.warn "[Aeternitas::PollJob] Aborting enqueue for #{pollable_meta_data_id} (job #{job.job_id}) due to existing lock: #{lock_digest}"
+          throw(:abort)
+        end
+      end
+    end
+
+    # === Retry Configuration (using standard ActiveJob) ===
+    retry_on StandardError,
+      attempts: MAX_TOTAL_ATTEMPTS,
+      wait: ->(executions) { execution_wait_time(executions) },
+      jitter: ->(executions) { [execution_wait_time(executions) * 0.1, 10.minutes].min } do |job, error|
+      handle_retries_exhausted(error)
+    end
+
+    # === GuardIsLocked Handling ===
+    rescue_from Aeternitas::Guard::GuardIsLocked do |error|
+      meta_data = Aeternitas::PollableMetaData.find_by(id: arguments.first)
+      return unless meta_data
+
+      pollable = meta_data.pollable
+      pollable_config = pollable.pollable_configuration
+      base_delay = (error.timeout - Time.now).to_f
+      meta_data.enqueue!
+
+      if pollable_config.sleep_on_guard_locked
+        if base_delay > 0
+          ActiveJob::Base.logger.warn "[Aeternitas::PollJob] Guard locked for #{arguments.first}. Sleep for #{base_delay.round(2)}s."
+          sleep(base_delay)
+        end
+        retry_job
+      else
+        guard_key_digest = self.class.generate_guard_key_digest(pollable)
+        lock_digest = self.class.generate_lock_digest(arguments.first)
+        unique_lock = Aeternitas::UniqueJobLock.find_by(lock_digest: lock_digest)
+
+        rank = if unique_lock
+          Aeternitas::UniqueJobLock.where(guard_key_digest: guard_key_digest)
+            .where("created_at <= ?", unique_lock.created_at)
+            .count
+        else
+          Aeternitas::UniqueJobLock.where(guard_key_digest: guard_key_digest).count
+        end
+
+        stagger_delay = rank * pollable.guard.cooldown.to_f
+        jitter = rand(0.0..2.0)
+        total_wait = base_delay + stagger_delay + jitter
+
+        if total_wait > 0
+          retry_job(wait: total_wait.seconds)
+          ActiveJob::Base.logger.info "[Aeternitas::PollJob] Guard locked for #{arguments.first}. Retry in #{total_wait.round(2)}s."
+        else
+          # GuardLock expired, retry with minimal delay
+          retry_job(wait: jitter.seconds)
+        end
+      end
+    end
+
+    def self.execution_wait_time(executions)
+      wait_index = executions - 1
+      RETRY_DELAYS[wait_index] || RETRY_DELAYS.last
+    end
+
+    def perform(pollable_meta_data_id)
+      meta_data = Aeternitas::PollableMetaData.find_by(id: pollable_meta_data_id)
+      if meta_data
+        pollable = meta_data.pollable
+        pollable&.execute_poll
+      else
+        ActiveJob::Base.logger.warn "[Aeternitas::PollJob] PollableMetaData with ID #{pollable_meta_data_id} not found."
+      end
+    end
+
+    after_perform -> { cleanup_lock("success") }
+
+    def self.generate_lock_digest(pollable_meta_data_id)
+      Digest::SHA256.hexdigest("#{name}:#{pollable_meta_data_id}")
+    end
+
+    def self.generate_guard_key_digest(pollable)
+      guard_key = pollable.pollable_configuration.guard_options[:key].call(pollable)
+      Digest::SHA256.hexdigest("guard-key:#{guard_key}")
+    end
+
+    def handle_retries_exhausted(error)
+      ActiveJob::Base.logger.error "[Aeternitas::PollJob] Retries exhausted for job #{job_id}. Error: #{error&.class} - #{error&.message}"
+      pollable_meta_data_id = arguments.first
+      meta_data = Aeternitas::PollableMetaData.find_by(id: pollable_meta_data_id)
+      meta_data&.disable_polling("Retries exhausted. Last error: #{error&.message}")
+      cleanup_lock("retries_exhausted")
+      raise error
+    end
+
+    def cleanup_lock(reason = "unknown")
+      return unless arguments.is_a?(Array) && arguments.first
+
+      pollable_meta_data_id = arguments.first
+      digest = self.class.generate_lock_digest(pollable_meta_data_id)
+      lock = Aeternitas::UniqueJobLock.find_by(lock_digest: digest)
+      lock&.destroy
+    end
+  end
+end