aeternitas 0.2.0 → 2.0.0

data/lib/aeternitas/poll_job.rb

@@ -0,0 +1,139 @@
+require_relative "aeternitas_job"
+require "digest"
+
+module Aeternitas
+  # ActiveJob worker responsible for executing the polling.
+  class PollJob < AeternitasJob
+    queue_as :polling
+
+    LOCK_EXPIRATION = 1.month
+    RETRY_DELAYS = [60.seconds, 1.hour, 1.day, 1.week].freeze
+    MAX_TOTAL_ATTEMPTS = 5
+
+    # === Job Uniqueness ===
+    before_enqueue do |job|
+      # Only check for uniqueness on the first attempt. Retries should not be blocked by their own lock.
+      if job.executions.zero?
+        pollable_meta_data_id = job.arguments.first
+        pollable_meta_data = Aeternitas::PollableMetaData.find(pollable_meta_data_id)
+        pollable = pollable_meta_data.pollable
+
+        lock_digest = self.class.generate_lock_digest(pollable_meta_data_id)
+        guard_key_digest = self.class.generate_guard_key_digest(pollable)
+
+        Aeternitas::UniqueJobLock.where("lock_digest = ? AND expires_at <= ?", lock_digest, Time.now).destroy_all
+
+        new_lock = Aeternitas::UniqueJobLock.new(
+          lock_digest: lock_digest,
+          guard_key_digest: guard_key_digest,
+          expires_at: Time.now + LOCK_EXPIRATION,
+          job_id: job.job_id
+        )
+
+        unless new_lock.save
+          ActiveJob::Base.logger.warn "[Aeternitas::PollJob] Aborting enqueue for #{pollable_meta_data_id} (job #{job.job_id}) due to existing lock: #{lock_digest}"
+          throw(:abort)
+        end
+      end
+    end
+
+    # === Retry Configuration (using standard ActiveJob) ===
+    retry_on StandardError,
+      attempts: MAX_TOTAL_ATTEMPTS,
+      wait: ->(executions) { execution_wait_time(executions) },
+      jitter: ->(executions) { [execution_wait_time(executions) * 0.1, 10.minutes].min } do |job, error|
+      handle_retries_exhausted(error)
+    end
+
+    # === GuardIsLocked Handling ===
+    rescue_from Aeternitas::Guard::GuardIsLocked do |error|
+      meta_data = Aeternitas::PollableMetaData.find_by(id: arguments.first)
+      return unless meta_data
+
+      pollable = meta_data.pollable
+      pollable_config = pollable.pollable_configuration
+      base_delay = (error.timeout - Time.now).to_f
+      meta_data.enqueue!
+
+      if pollable_config.sleep_on_guard_locked
+        delay = Aeternitas.test_mode? ? 0 : base_delay
+        if delay > 0
+          ActiveJob::Base.logger.warn "[Aeternitas::PollJob] Guard locked for #{arguments.first}. Sleep for #{base_delay.round(2)}s."
+          sleep(delay)
+        end
+        retry_job
+      else
+        guard_key_digest = self.class.generate_guard_key_digest(pollable)
+        lock_digest = self.class.generate_lock_digest(arguments.first)
+        unique_lock = Aeternitas::UniqueJobLock.find_by(lock_digest: lock_digest)
+
+        rank = if unique_lock
+          Aeternitas::UniqueJobLock.where(guard_key_digest: guard_key_digest)
+            .where("created_at <= ?", unique_lock.created_at)
+            .count
+        else
+          Aeternitas::UniqueJobLock.where(guard_key_digest: guard_key_digest).count
+        end
+
+        stagger_delay = rank * pollable.guard.cooldown.to_f
+        jitter = rand(0.0..2.0)
+        total_wait = base_delay + stagger_delay + jitter
+
+        if total_wait > 0 || Aeternitas.test_mode?
+          wait_time = Aeternitas.test_mode? ? 0.seconds : total_wait.seconds
+          retry_job(wait: wait_time)
+          ActiveJob::Base.logger.info "[Aeternitas::PollJob] Guard locked for #{arguments.first}. Retry in #{total_wait.round(2)}s."
+        else
+          # GuardLock expired, retry with minimal delay
+          wait_time = Aeternitas.test_mode? ? 0.seconds : jitter.seconds
+          retry_job(wait: wait_time)
+        end
+      end
+    end
+
+    def self.execution_wait_time(executions)
+      return 0.seconds if Aeternitas.test_mode?
+      wait_index = executions - 1
+      RETRY_DELAYS[wait_index] || RETRY_DELAYS.last
+    end
+
+    def perform(pollable_meta_data_id)
+      meta_data = Aeternitas::PollableMetaData.find_by(id: pollable_meta_data_id)
+      if meta_data
+        pollable = meta_data.pollable
+        pollable&.execute_poll
+      else
+        ActiveJob::Base.logger.warn "[Aeternitas::PollJob] PollableMetaData with ID #{pollable_meta_data_id} not found."
+      end
+    end
+
+    after_perform -> { cleanup_lock("success") }
+
+    def self.generate_lock_digest(pollable_meta_data_id)
+      Digest::SHA256.hexdigest("#{name}:#{pollable_meta_data_id}")
+    end
+
+    def self.generate_guard_key_digest(pollable)
+      guard_key = pollable.pollable_configuration.guard_options[:key].call(pollable)
+      Digest::SHA256.hexdigest("guard-key:#{guard_key}")
+    end
+
+    def handle_retries_exhausted(error)
+      ActiveJob::Base.logger.error "[Aeternitas::PollJob] Retries exhausted for job #{job_id}. Error: #{error&.class} - #{error&.message}"
+      pollable_meta_data_id = arguments.first
+      meta_data = Aeternitas::PollableMetaData.find_by(id: pollable_meta_data_id)
+      meta_data&.disable_polling("Retries exhausted. Last error: #{error&.message}")
+      cleanup_lock("retries_exhausted")
+      raise error
+    end
+
+    def cleanup_lock(reason = "unknown")
+      return unless arguments.is_a?(Array) && arguments.first
+
+      pollable_meta_data_id = arguments.first
+      digest = self.class.generate_lock_digest(pollable_meta_data_id)
+      lock = Aeternitas::UniqueJobLock.find_by(lock_digest: digest)
+      lock&.destroy
+    end
+  end
+end

data/lib/aeternitas/pollable/configuration.rb

@@ -9,7 +9,7 @@ module Aeternitas
     # @!attribute [rw] after_polling
     #   Methods to be run after each successful poll
     # @!attribute [rw] queue
-    #   Sidekiq queue the poll job will be enqueued in (Default: 'polling')
+    #   The queue the poll job will be enqueued in (Default: 'polling')
     # @!attribute [rw] guard_options
     #   Configuration of the pollables lock (Default: key => class+id, cooldown => 5.seconds, timeout => 10.minutes)
     # @!attribute [rw] deactivation_errors
@@ -18,17 +18,17 @@ module Aeternitas
     #   Errors in this list will be wrapped by {Aeternitas::Error::Ignored} if they occur while polling
     #   (i.e. ignore in your exception tracker)
     # @!attribute [rw] sleep_on_guard_locked
-    #   When set to true poll jobs (and effectively the Sidekiq worker thread) will sleep until the
-    #   lock is released if the lock could not be acquired. (Default: true)
+    #   When set to true, the ActiveJob worker thread will sleep until the
+    #   lock is released if the lock could not be acquired. (Default: false)
     class Configuration
       attr_accessor :deactivation_errors,
-                    :before_polling,
-                    :queue,
-                    :polling_frequency,
-                    :after_polling,
-                    :guard_options,
-                    :ignored_errors,
-                    :sleep_on_guard_locked
+        :before_polling,
+        :queue,
+        :polling_frequency,
+        :after_polling,
+        :guard_options,
+        :ignored_errors,
+        :sleep_on_guard_locked
 
       # Creates a new Configuration with default options
       def initialize
@@ -36,29 +36,28 @@ module Aeternitas
         @before_polling = []
         @after_polling = []
         @guard_options = {
-          key: ->(obj) { return obj.class.name.to_s },
+          key: ->(obj) { obj.class.name.to_s },
           timeout: 10.minutes,
           cooldown: 5.seconds
         }
         @deactivation_errors = []
         @ignored_errors = []
-        @queue = 'polling'
-        @sleep_on_guard_locked = true
+        @queue = "polling"
+        @sleep_on_guard_locked = false
       end
 
       def copy
         config = Configuration.new
-        config.polling_frequency = self.polling_frequency
-        config.before_polling = self.before_polling.deep_dup
-        config.after_polling = self.after_polling.deep_dup
-        config.guard_options = self.guard_options.deep_dup
-        config.deactivation_errors = self.deactivation_errors.deep_dup
-        config.ignored_errors = self.ignored_errors.deep_dup
-        config.queue = self.queue
-        config.sleep_on_guard_locked = self.sleep_on_guard_locked
+        config.polling_frequency = polling_frequency
+        config.before_polling = before_polling.deep_dup
+        config.after_polling = after_polling.deep_dup
+        config.guard_options = guard_options.deep_dup
+        config.deactivation_errors = deactivation_errors.deep_dup
+        config.ignored_errors = ignored_errors.deep_dup
+        config.queue = queue
+        config.sleep_on_guard_locked = sleep_on_guard_locked
         config
       end
     end
   end
 end
-

data/lib/aeternitas/pollable/dsl.rb

@@ -21,10 +21,10 @@ module Aeternitas
       #   polling_frequency ->(pollable) {Time.now + 1.month + Time.now - pollable.created_at.to_i / 3.month * 1.month}
       # @todo allow custom methods via reference
       def polling_frequency(frequency)
-        if frequency.is_a?(Symbol)
-          @configuration.polling_frequency = Aeternitas::PollingFrequency.by_name(frequency)
+        @configuration.polling_frequency = if frequency.is_a?(Symbol)
+          Aeternitas::PollingFrequency.by_name(frequency)
         else
-          @configuration.polling_frequency = frequency
+          frequency
         end
       end
 
@@ -38,10 +38,10 @@ module Aeternitas
       # @example method by block
       #   before_polling ->(pollable) {do_something}
       def before_polling(method)
-        if method.is_a?(Symbol)
-          @configuration.before_polling << ->(pollable) { pollable.send(method) }
+        @configuration.before_polling << if method.is_a?(Symbol)
+          ->(pollable) { pollable.send(method) }
         else
-          @configuration.before_polling << method
+          method
         end
       end
 
@@ -55,10 +55,10 @@ module Aeternitas
       # @example method by block
       #   after_polling ->(pollable) {do_something}
       def after_polling(method)
-        if method.is_a?(Symbol)
-          @configuration.after_polling << ->(pollable) { pollable.send(method) }
+        @configuration.after_polling << if method.is_a?(Symbol)
+          ->(pollable) { pollable.send(method) }
         else
-          @configuration.after_polling << method
+          method
         end
       end
 
@@ -94,13 +94,13 @@ module Aeternitas
       #   guard_key ->(pollable) {URI.parse(pollable.url).host}
       def guard_key(key)
         @configuration.guard_options[:key] = case key
-                                when Symbol
-                                  ->(obj) { return obj.send(key) }
-                                when Proc
-                                  key
-                                else
-                                  ->(obj) { return key.to_s }
-                                end
+        when Symbol
+          ->(obj) { obj.send(key) }
+        when Proc
+          key
+        else
+          ->(obj) { key.to_s }
+        end
       end
 
       # Configure the guard.
@@ -121,4 +121,3 @@ module Aeternitas
     end
   end
 end
-

data/lib/aeternitas/pollable.rb

@@ -1,5 +1,5 @@
-require 'aeternitas/pollable/configuration'
-require 'aeternitas/pollable/dsl'
+require "aeternitas/pollable/configuration"
+require "aeternitas/pollable/dsl"
 
 module Aeternitas
   # Mixin that enables the frequent polling of the receiving class.
@@ -25,20 +25,20 @@ module Aeternitas
     extend ActiveSupport::Concern
 
     included do
-      raise StandardError, 'Aeternitas::Pollable must inherit from ActiveRecord::Base' unless self.ancestors.include?(ActiveRecord::Base)
+      raise StandardError, "Aeternitas::Pollable must inherit from ActiveRecord::Base" unless ancestors.include?(ActiveRecord::Base)
 
       has_one :pollable_meta_data, as: :pollable,
-              dependent: :destroy,
-              class_name: 'Aeternitas::PollableMetaData'
+        dependent: :destroy,
+        class_name: "Aeternitas::PollableMetaData"
 
       has_many :sources, as: :pollable,
-               dependent: :destroy,
-               class_name: 'Aeternitas::Source'
+        dependent: :destroy,
+        class_name: "Aeternitas::Source"
 
       validates :pollable_meta_data, presence: true
 
       before_validation ->(pollable) do
-        pollable.pollable_meta_data ||= pollable.build_pollable_meta_data(state: 'waiting')
+        pollable.pollable_meta_data ||= pollable.build_pollable_meta_data(state: "waiting")
         pollable.pollable_meta_data.pollable_class = pollable.class.name
       end
 
@@ -53,7 +53,10 @@ module Aeternitas
 
       begin
         guard.with_lock { poll }
-      rescue StandardError => e
+      rescue Aeternitas::Guard::GuardIsLocked
+        # Do not transition to the 'errored' state for a guard lock.
+        raise
+      rescue => e
         if pollable_configuration.deactivation_errors.include?(e.class)
           disable_polling(e)
           return false
@@ -67,7 +70,7 @@ module Aeternitas
       end
 
       _after_poll
-    rescue StandardError => e
+    rescue => e
       begin
         log_poll_error(e)
       ensure
@@ -75,8 +78,6 @@ module Aeternitas
       end
     end
 
-
-
     # This method implements the class specific polling behaviour.
     # It is only called after the lock was acquired successfully.
     #
@@ -91,8 +92,8 @@ module Aeternitas
     #   {Aeternitas::Pollable} was included. Otherwise it is done automatically after creation.
     def register_pollable
       self.pollable_meta_data ||= create_pollable_meta_data(
-          state: 'waiting',
-          pollable_class: self.class.name
+        state: "waiting",
+        pollable_class: self.class.name
       )
     end
 
@@ -121,7 +122,7 @@ module Aeternitas
     # @param [String] raw_content the sources raw content
     # @return [Aeternitas::Source] the newly created or existing source
     def add_source(raw_content)
-      source = self.sources.create(raw_content: raw_content)
+      source = sources.create(raw_content: raw_content)
       return nil unless source.persisted?
 
       Aeternitas::Metrics.log(:sources_created, self.class)
@@ -142,7 +143,7 @@ module Aeternitas
     # Run all postpolling methods
     def _after_poll
       pollable_meta_data.wait! do
-        pollable_meta_data.update_attributes!(
+        pollable_meta_data.update!(
           last_polling: Time.now,
           next_polling: pollable_configuration.polling_frequency.call(self)
         )
@@ -185,7 +186,7 @@ module Aeternitas
       # Configure the polling process.
       # For available configuration options see {Aeternitas::Pollable::Configuration} and {Aeternitas::Pollable::DSL}
       def polling_options(&block)
-        Aeternitas::Pollable::Dsl.new(self.pollable_configuration, &block)
+        Aeternitas::Pollable::Dsl.new(pollable_configuration, &block)
       end
 
       def inherited(other)
@@ -194,4 +195,4 @@ module Aeternitas
       end
     end
   end
-end
+end

data/lib/aeternitas/pollable_meta_data.rb

@@ -1,10 +1,10 @@
-require 'aasm'
+require "aasm"
 
 module Aeternitas
   # Stores the meta data of all pollables
   # Every pollable needs to have exactly one meta data object
   class PollableMetaData < ActiveRecord::Base
-    self.table_name = 'aeternitas_pollable_meta_data'
+    self.table_name = "aeternitas_pollable_meta_data"
 
     include AASM
     ######
@@ -25,20 +25,29 @@ module Aeternitas
 
     belongs_to :pollable, polymorphic: true
 
-    validates :pollable_type, presence: true, uniqueness: { scope: :pollable_id }
-    validates :pollable_id, presence: true, uniqueness: { scope: :pollable_type }
+    validates :pollable_type, presence: true, uniqueness: {scope: :pollable_id}
+    validates :pollable_id, presence: true, uniqueness: {scope: :pollable_type}
     validates :pollable_class, presence: true
     validates :next_polling, presence: true
 
     aasm column: :state do
+      # Only pollables in this state are picked up for enqueueing ('next_polling')
       state :waiting, initial: true
+
+      # A PollJob has been submitted and the pollable is waiting for a worker to start processing.
       state :enqueued
+
+      # A worker has picked up the job and is currently executing the pollable.
       state :active
+
+      # Polling has been permanently disabled (e.g., due to retries exhausted).
       state :deactivated
+
+      # A generic error occurred during polling. The job will be retried.
       state :errored
 
       event :enqueue do
-        transitions from: %i[waiting deactivated errored], to: :enqueued
+        transitions from: %i[waiting deactivated errored active], to: :enqueued
       end
 
       event :poll do
@@ -58,16 +67,16 @@ module Aeternitas
       end
     end
 
-    scope(:due, ->() { waiting.where('next_polling < ?', Time.now) })
+    scope(:due, -> { waiting.where("next_polling < ?", Time.now) })
 
     # Disables polling of this instance
     #
     # @param [String] reason Reason for the deactivation. (E.g. an error message)
     def disable_polling(reason = nil)
-      self.deactivate
+      deactivate
       self.deactivation_reason = reason.to_s
       self.deactivated_at = Time.now
-      self.save!
+      save!
     end
   end
-end
+end

data/lib/aeternitas/polling_frequency.rb

@@ -1,9 +1,9 @@
 module Aeternitas
   # Stores default polling frequency calculation methods.
   module PollingFrequency
-    HOURLY  = ->(context) { Time.now + 1.hour }
-    DAILY   = ->(context) { Time.now + 1.day  }
-    WEEKLY  = ->(context) { Time.now + 1.week }
+    HOURLY = ->(context) { Time.now + 1.hour }
+    DAILY = ->(context) { Time.now + 1.day }
+    WEEKLY = ->(context) { Time.now + 1.week }
     MONTHLY = ->(context) { Time.now + 1.month }
 
     # Retrieves the build-in polling frequency methods by name.
@@ -21,4 +21,4 @@ module Aeternitas
       end
     end
   end
-end
+end

data/lib/aeternitas/source.rb

@@ -13,7 +13,7 @@ module Aeternitas
     # end
     # add_index :aeternitas_sources, [:pollable_id, :pollable_type], name: 'aeternitas_pollable_source'
     ######
-    self.table_name = 'aeternitas_sources'
+    self.table_name = "aeternitas_sources"
 
     attr_writer :raw_content
 
@@ -42,21 +42,21 @@ module Aeternitas
     # Get the sources raw content.
     # @return [String] the sources raw content
     def raw_content
-      @raw_content ||= Aeternitas.config.get_storage_adapter.retrieve(self.fingerprint)
+      @raw_content ||= Aeternitas.config.get_storage_adapter.retrieve(fingerprint)
     end
 
     private
 
     def create_file
-      Aeternitas.config.get_storage_adapter.store(self.fingerprint, raw_content)
+      Aeternitas.config.get_storage_adapter.store(fingerprint, raw_content)
     end
 
     def delete_file
-      Aeternitas.config.get_storage_adapter.delete(self.fingerprint)
+      Aeternitas.config.get_storage_adapter.delete(fingerprint)
     end
 
     def ensure_fingerprint
       self.fingerprint ||= generate_fingerprint
     end
   end
-end
+end

data/lib/aeternitas/storage_adapter/file.rb

@@ -2,7 +2,6 @@ module Aeternitas
   class StorageAdapter
     # A storage adapter that stores the entries on disk.
     class File < Aeternitas::StorageAdapter
-
       # Create a new File storage adapter.
       # @param [Hash] config the adapters config
       # @option config [String] :directory specifies where the entries are stored
@@ -14,22 +13,20 @@ module Aeternitas
         path = file_path(id)
         ensure_folders_exist(path)
         raise(Aeternitas::Errors::SourceDataExists, id) if ::File.exist?(path)
-        ::File.open(path, 'w+', encoding: 'ascii-8bit') do |f|
+        ::File.open(path, "w+", encoding: "ascii-8bit") do |f|
           f.write(Zlib.deflate(raw_content, Zlib::BEST_COMPRESSION))
         end
       end
 
       def retrieve(id)
         raise(Aeternitas::Errors::SourceDataNotFound, id) unless exist?(id)
-        Zlib.inflate(::File.read(file_path(id), encoding: 'ascii-8bit'))
+        Zlib.inflate(::File.read(file_path(id), encoding: "ascii-8bit"))
       end
 
       def delete(id)
-        begin
-          !!::File.delete(file_path(id))
-        rescue Errno::ENOENT => e
-          return false
-        end
+        !!::File.delete(file_path(id))
+      rescue Errno::ENOENT
+        false
       end
 
       def exist?(id)
@@ -57,9 +54,9 @@ module Aeternitas
       # @return [String] the entries location
       def file_path(id)
         ::File.join(
-            @config[:directory],
-            id[0..1], id[2..3], id[4..5],
-            id[6..-1]
+          @config[:directory],
+          id[0..1], id[2..3], id[4..5],
+          id[6..]
         )
       end
 
@@ -70,4 +67,4 @@ module Aeternitas
       end
     end
   end
-end
+end

data/lib/aeternitas/storage_adapter.rb

@@ -3,7 +3,6 @@ module Aeternitas
   # Storage Adapters take care of handling source files.
   # @abstract Create a subclass and override {#store}, #{retrieve} and #{#delete} to create a new storage adapter
   class StorageAdapter
-
     # Create a new storage adapter
     # @param [Hash] config the adapters configuration
     def initialize(config)
@@ -41,6 +40,5 @@ module Aeternitas
     def exist?(id)
       raise NotImplementedError, "#{self.class.name} does not implement #exist?, required by Aeternitas::StorageAdapter"
     end
-
   end
-end
+end

data/lib/aeternitas/test.rb

@@ -0,0 +1,13 @@
+module Aeternitas
+  # Provides test helpers for aeternitas
+  module Test
+    # Executes a block of code in test mode; all cooldowns and wait times are set to 0.
+    def self.test_mode
+      original_mode = Aeternitas.test_mode?
+      Aeternitas.test_mode = true
+      yield
+    ensure
+      Aeternitas.test_mode = original_mode
+    end
+  end
+end

data/lib/aeternitas/unique_job_lock.rb

@@ -0,0 +1,15 @@
+require "active_record"
+
+module Aeternitas
+  # Since ActiveJob lacks a built-in uniqueness feature, this model prevents duplicate jobs from running simultaneously.
+  # This is achieved using the unique key `lock_digest`, which is calculated for each job instance.
+  # Additionally, this model stores `guard_key_digest` — a hash of the pollable's guard key —
+  # to determine how many jobs are waiting on the same resource and to adjust retries accordingly.
+  # Finally, `expires_at` is used to clean up stale locks left by crashed workers.
+  class UniqueJobLock < ActiveRecord::Base
+    self.table_name = "aeternitas_unique_job_locks"
+
+    validates :lock_digest, presence: true, uniqueness: true
+    validates :expires_at, presence: true
+  end
+end

data/lib/aeternitas/version.rb

@@ -1,3 +1,3 @@
 module Aeternitas
-  VERSION = "0.2.0"
+  VERSION = "2.0.0"
 end