aeternitas 0.2.0 → 2.0.0.rc1

data/lib/aeternitas/pollable/configuration.rb

@@ -9,7 +9,7 @@ module Aeternitas
     # @!attribute [rw] after_polling
     #   Methods to be run after each successful poll
     # @!attribute [rw] queue
-    #   Sidekiq queue the poll job will be enqueued in (Default: 'polling')
+    #   The queue the poll job will be enqueued in (Default: 'polling')
     # @!attribute [rw] guard_options
     #   Configuration of the pollables lock (Default: key => class+id, cooldown => 5.seconds, timeout => 10.minutes)
     # @!attribute [rw] deactivation_errors
@@ -18,17 +18,17 @@ module Aeternitas
     #   Errors in this list will be wrapped by {Aeternitas::Error::Ignored} if they occur while polling
     #   (i.e. ignore in your exception tracker)
     # @!attribute [rw] sleep_on_guard_locked
-    #   When set to true poll jobs (and effectively the Sidekiq worker thread) will sleep until the
-    #   lock is released if the lock could not be acquired. (Default: true)
+    #   When set to true, the ActiveJob worker thread will sleep until the
+    #   lock is released if the lock could not be acquired. (Default: false)
     class Configuration
       attr_accessor :deactivation_errors,
-                    :before_polling,
-                    :queue,
-                    :polling_frequency,
-                    :after_polling,
-                    :guard_options,
-                    :ignored_errors,
-                    :sleep_on_guard_locked
+        :before_polling,
+        :queue,
+        :polling_frequency,
+        :after_polling,
+        :guard_options,
+        :ignored_errors,
+        :sleep_on_guard_locked
 
       # Creates a new Configuration with default options
       def initialize
@@ -36,29 +36,28 @@ module Aeternitas
         @before_polling = []
         @after_polling = []
         @guard_options = {
-          key: ->(obj) { return obj.class.name.to_s },
+          key: ->(obj) { obj.class.name.to_s },
           timeout: 10.minutes,
           cooldown: 5.seconds
         }
         @deactivation_errors = []
         @ignored_errors = []
-        @queue = 'polling'
-        @sleep_on_guard_locked = true
+        @queue = "polling"
+        @sleep_on_guard_locked = false
       end
 
       def copy
         config = Configuration.new
-        config.polling_frequency = self.polling_frequency
-        config.before_polling = self.before_polling.deep_dup
-        config.after_polling = self.after_polling.deep_dup
-        config.guard_options = self.guard_options.deep_dup
-        config.deactivation_errors = self.deactivation_errors.deep_dup
-        config.ignored_errors = self.ignored_errors.deep_dup
-        config.queue = self.queue
-        config.sleep_on_guard_locked = self.sleep_on_guard_locked
+        config.polling_frequency = polling_frequency
+        config.before_polling = before_polling.deep_dup
+        config.after_polling = after_polling.deep_dup
+        config.guard_options = guard_options.deep_dup
+        config.deactivation_errors = deactivation_errors.deep_dup
+        config.ignored_errors = ignored_errors.deep_dup
+        config.queue = queue
+        config.sleep_on_guard_locked = sleep_on_guard_locked
         config
       end
     end
   end
 end
-

data/lib/aeternitas/pollable/dsl.rb

@@ -21,10 +21,10 @@ module Aeternitas
       #   polling_frequency ->(pollable) {Time.now + 1.month + Time.now - pollable.created_at.to_i / 3.month * 1.month}
       # @todo allow custom methods via reference
       def polling_frequency(frequency)
-        if frequency.is_a?(Symbol)
-          @configuration.polling_frequency = Aeternitas::PollingFrequency.by_name(frequency)
+        @configuration.polling_frequency = if frequency.is_a?(Symbol)
+          Aeternitas::PollingFrequency.by_name(frequency)
         else
-          @configuration.polling_frequency = frequency
+          frequency
         end
       end
 
@@ -38,10 +38,10 @@ module Aeternitas
       # @example method by block
       #   before_polling ->(pollable) {do_something}
       def before_polling(method)
-        if method.is_a?(Symbol)
-          @configuration.before_polling << ->(pollable) { pollable.send(method) }
+        @configuration.before_polling << if method.is_a?(Symbol)
+          ->(pollable) { pollable.send(method) }
         else
-          @configuration.before_polling << method
+          method
         end
       end
 
@@ -55,10 +55,10 @@ module Aeternitas
       # @example method by block
       #   after_polling ->(pollable) {do_something}
       def after_polling(method)
-        if method.is_a?(Symbol)
-          @configuration.after_polling << ->(pollable) { pollable.send(method) }
+        @configuration.after_polling << if method.is_a?(Symbol)
+          ->(pollable) { pollable.send(method) }
         else
-          @configuration.after_polling << method
+          method
         end
       end
 
@@ -94,13 +94,13 @@ module Aeternitas
       #   guard_key ->(pollable) {URI.parse(pollable.url).host}
       def guard_key(key)
         @configuration.guard_options[:key] = case key
-                                when Symbol
-                                  ->(obj) { return obj.send(key) }
-                                when Proc
-                                  key
-                                else
-                                  ->(obj) { return key.to_s }
-                                end
+        when Symbol
+          ->(obj) { obj.send(key) }
+        when Proc
+          key
+        else
+          ->(obj) { key.to_s }
+        end
       end
 
       # Configure the guard.
@@ -121,4 +121,3 @@ module Aeternitas
     end
   end
 end
-

data/lib/aeternitas/pollable.rb

@@ -1,5 +1,5 @@
-require 'aeternitas/pollable/configuration'
-require 'aeternitas/pollable/dsl'
+require "aeternitas/pollable/configuration"
+require "aeternitas/pollable/dsl"
 
 module Aeternitas
   # Mixin that enables the frequent polling of the receiving class.
@@ -25,20 +25,20 @@ module Aeternitas
     extend ActiveSupport::Concern
 
     included do
-      raise StandardError, 'Aeternitas::Pollable must inherit from ActiveRecord::Base' unless self.ancestors.include?(ActiveRecord::Base)
+      raise StandardError, "Aeternitas::Pollable must inherit from ActiveRecord::Base" unless ancestors.include?(ActiveRecord::Base)
 
       has_one :pollable_meta_data, as: :pollable,
-              dependent: :destroy,
-              class_name: 'Aeternitas::PollableMetaData'
+        dependent: :destroy,
+        class_name: "Aeternitas::PollableMetaData"
 
       has_many :sources, as: :pollable,
-               dependent: :destroy,
-               class_name: 'Aeternitas::Source'
+        dependent: :destroy,
+        class_name: "Aeternitas::Source"
 
       validates :pollable_meta_data, presence: true
 
       before_validation ->(pollable) do
-        pollable.pollable_meta_data ||= pollable.build_pollable_meta_data(state: 'waiting')
+        pollable.pollable_meta_data ||= pollable.build_pollable_meta_data(state: "waiting")
         pollable.pollable_meta_data.pollable_class = pollable.class.name
       end
 
@@ -53,7 +53,10 @@ module Aeternitas
 
       begin
         guard.with_lock { poll }
-      rescue StandardError => e
+      rescue Aeternitas::Guard::GuardIsLocked
+        # Do not transition to the 'errored' state for a guard lock.
+        raise
+      rescue => e
         if pollable_configuration.deactivation_errors.include?(e.class)
           disable_polling(e)
           return false
@@ -67,7 +70,7 @@ module Aeternitas
       end
 
       _after_poll
-    rescue StandardError => e
+    rescue => e
       begin
         log_poll_error(e)
       ensure
@@ -75,8 +78,6 @@ module Aeternitas
       end
     end
 
-
-
     # This method implements the class specific polling behaviour.
     # It is only called after the lock was acquired successfully.
     #
@@ -91,8 +92,8 @@ module Aeternitas
     #   {Aeternitas::Pollable} was included. Otherwise it is done automatically after creation.
     def register_pollable
       self.pollable_meta_data ||= create_pollable_meta_data(
-          state: 'waiting',
-          pollable_class: self.class.name
+        state: "waiting",
+        pollable_class: self.class.name
       )
     end
 
@@ -121,7 +122,7 @@ module Aeternitas
     # @param [String] raw_content the sources raw content
     # @return [Aeternitas::Source] the newly created or existing source
     def add_source(raw_content)
-      source = self.sources.create(raw_content: raw_content)
+      source = sources.create(raw_content: raw_content)
       return nil unless source.persisted?
 
       Aeternitas::Metrics.log(:sources_created, self.class)
@@ -142,7 +143,7 @@ module Aeternitas
     # Run all postpolling methods
     def _after_poll
       pollable_meta_data.wait! do
-        pollable_meta_data.update_attributes!(
+        pollable_meta_data.update!(
           last_polling: Time.now,
           next_polling: pollable_configuration.polling_frequency.call(self)
         )
@@ -185,7 +186,7 @@ module Aeternitas
       # Configure the polling process.
       # For available configuration options see {Aeternitas::Pollable::Configuration} and {Aeternitas::Pollable::DSL}
       def polling_options(&block)
-        Aeternitas::Pollable::Dsl.new(self.pollable_configuration, &block)
+        Aeternitas::Pollable::Dsl.new(pollable_configuration, &block)
       end
 
       def inherited(other)
@@ -194,4 +195,4 @@ module Aeternitas
       end
     end
   end
-end
+end

data/lib/aeternitas/pollable_meta_data.rb

@@ -1,10 +1,10 @@
-require 'aasm'
+require "aasm"
 
 module Aeternitas
   # Stores the meta data of all pollables
   # Every pollable needs to have exactly one meta data object
   class PollableMetaData < ActiveRecord::Base
-    self.table_name = 'aeternitas_pollable_meta_data'
+    self.table_name = "aeternitas_pollable_meta_data"
 
     include AASM
     ######
@@ -25,20 +25,29 @@ module Aeternitas
 
     belongs_to :pollable, polymorphic: true
 
-    validates :pollable_type, presence: true, uniqueness: { scope: :pollable_id }
-    validates :pollable_id, presence: true, uniqueness: { scope: :pollable_type }
+    validates :pollable_type, presence: true, uniqueness: {scope: :pollable_id}
+    validates :pollable_id, presence: true, uniqueness: {scope: :pollable_type}
     validates :pollable_class, presence: true
     validates :next_polling, presence: true
 
     aasm column: :state do
+      # Only pollables in this state are picked up for enqueueing ('next_polling')
       state :waiting, initial: true
+
+      # A PollJob has been submitted and the pollable is waiting for a worker to start processing.
       state :enqueued
+
+      # A worker has picked up the job and is currently executing the pollable.
       state :active
+
+      # Polling has been permanently disabled (e.g., due to retries exhausted).
       state :deactivated
+
+      # A generic error occurred during polling. The job will be retried.
       state :errored
 
       event :enqueue do
-        transitions from: %i[waiting deactivated errored], to: :enqueued
+        transitions from: %i[waiting deactivated errored active], to: :enqueued
       end
 
       event :poll do
@@ -58,16 +67,16 @@ module Aeternitas
       end
     end
 
-    scope(:due, ->() { waiting.where('next_polling < ?', Time.now) })
+    scope(:due, -> { waiting.where("next_polling < ?", Time.now) })
 
     # Disables polling of this instance
     #
     # @param [String] reason Reason for the deactivation. (E.g. an error message)
     def disable_polling(reason = nil)
-      self.deactivate
+      deactivate
       self.deactivation_reason = reason.to_s
       self.deactivated_at = Time.now
-      self.save!
+      save!
     end
   end
-end
+end

data/lib/aeternitas/polling_frequency.rb

@@ -1,9 +1,9 @@
 module Aeternitas
   # Stores default polling frequency calculation methods.
   module PollingFrequency
-    HOURLY  = ->(context) { Time.now + 1.hour }
-    DAILY   = ->(context) { Time.now + 1.day  }
-    WEEKLY  = ->(context) { Time.now + 1.week }
+    HOURLY = ->(context) { Time.now + 1.hour }
+    DAILY = ->(context) { Time.now + 1.day }
+    WEEKLY = ->(context) { Time.now + 1.week }
     MONTHLY = ->(context) { Time.now + 1.month }
 
     # Retrieves the build-in polling frequency methods by name.
@@ -21,4 +21,4 @@ module Aeternitas
       end
     end
   end
-end
+end

data/lib/aeternitas/source.rb

@@ -13,7 +13,7 @@ module Aeternitas
     # end
     # add_index :aeternitas_sources, [:pollable_id, :pollable_type], name: 'aeternitas_pollable_source'
     ######
-    self.table_name = 'aeternitas_sources'
+    self.table_name = "aeternitas_sources"
 
     attr_writer :raw_content
 
@@ -42,21 +42,21 @@ module Aeternitas
     # Get the sources raw content.
     # @return [String] the sources raw content
     def raw_content
-      @raw_content ||= Aeternitas.config.get_storage_adapter.retrieve(self.fingerprint)
+      @raw_content ||= Aeternitas.config.get_storage_adapter.retrieve(fingerprint)
     end
 
     private
 
     def create_file
-      Aeternitas.config.get_storage_adapter.store(self.fingerprint, raw_content)
+      Aeternitas.config.get_storage_adapter.store(fingerprint, raw_content)
     end
 
     def delete_file
-      Aeternitas.config.get_storage_adapter.delete(self.fingerprint)
+      Aeternitas.config.get_storage_adapter.delete(fingerprint)
     end
 
     def ensure_fingerprint
       self.fingerprint ||= generate_fingerprint
     end
   end
-end
+end

data/lib/aeternitas/storage_adapter/file.rb

@@ -2,7 +2,6 @@ module Aeternitas
   class StorageAdapter
     # A storage adapter that stores the entries on disk.
     class File < Aeternitas::StorageAdapter
-
       # Create a new File storage adapter.
       # @param [Hash] config the adapters config
       # @option config [String] :directory specifies where the entries are stored
@@ -14,22 +13,20 @@ module Aeternitas
         path = file_path(id)
         ensure_folders_exist(path)
         raise(Aeternitas::Errors::SourceDataExists, id) if ::File.exist?(path)
-        ::File.open(path, 'w+', encoding: 'ascii-8bit') do |f|
+        ::File.open(path, "w+", encoding: "ascii-8bit") do |f|
           f.write(Zlib.deflate(raw_content, Zlib::BEST_COMPRESSION))
         end
       end
 
       def retrieve(id)
         raise(Aeternitas::Errors::SourceDataNotFound, id) unless exist?(id)
-        Zlib.inflate(::File.read(file_path(id), encoding: 'ascii-8bit'))
+        Zlib.inflate(::File.read(file_path(id), encoding: "ascii-8bit"))
       end
 
       def delete(id)
-        begin
-          !!::File.delete(file_path(id))
-        rescue Errno::ENOENT => e
-          return false
-        end
+        !!::File.delete(file_path(id))
+      rescue Errno::ENOENT
+        false
       end
 
       def exist?(id)
@@ -57,9 +54,9 @@ module Aeternitas
       # @return [String] the entries location
       def file_path(id)
         ::File.join(
-            @config[:directory],
-            id[0..1], id[2..3], id[4..5],
-            id[6..-1]
+          @config[:directory],
+          id[0..1], id[2..3], id[4..5],
+          id[6..]
         )
       end
 
@@ -70,4 +67,4 @@ module Aeternitas
       end
     end
   end
-end
+end

data/lib/aeternitas/storage_adapter.rb

@@ -3,7 +3,6 @@ module Aeternitas
   # Storage Adapters take care of handling source files.
   # @abstract Create a subclass and override {#store}, #{retrieve} and #{#delete} to create a new storage adapter
   class StorageAdapter
-
     # Create a new storage adapter
     # @param [Hash] config the adapters configuration
     def initialize(config)
@@ -41,6 +40,5 @@ module Aeternitas
     def exist?(id)
       raise NotImplementedError, "#{self.class.name} does not implement #exist?, required by Aeternitas::StorageAdapter"
     end
-
   end
-end
+end

data/lib/aeternitas/unique_job_lock.rb

@@ -0,0 +1,15 @@
+require "active_record"
+
+module Aeternitas
+  # Since ActiveJob lacks a built-in uniqueness feature, this model prevents duplicate jobs from running simultaneously.
+  # This is achieved using the unique key `lock_digest`, which is calculated for each job instance.
+  # Additionally, this model stores `guard_key_digest` — a hash of the pollable's guard key —
+  # to determine how many jobs are waiting on the same resource and to adjust retries accordingly.
+  # Finally, `expires_at` is used to clean up stale locks left by crashed workers.
+  class UniqueJobLock < ActiveRecord::Base
+    self.table_name = "aeternitas_unique_job_locks"
+
+    validates :lock_digest, presence: true, uniqueness: true
+    validates :expires_at, presence: true
+  end
+end

data/lib/aeternitas/version.rb

@@ -1,3 +1,3 @@
 module Aeternitas
-  VERSION = "0.2.0"
+  VERSION = "2.0.0.rc1"
 end

data/lib/aeternitas.rb

@@ -1,8 +1,5 @@
+require "ostruct"
 require "active_support/all"
-require "redis"
-require "connection_pool"
-require "sidekiq-unique-jobs"
-require "tabs_tabs"
 require "aeternitas/version"
 require "aeternitas/guard"
 require "aeternitas/pollable"
@@ -11,18 +8,18 @@ require "aeternitas/source"
 require "aeternitas/polling_frequency"
 require "aeternitas/errors"
 require "aeternitas/storage_adapter"
-require "aeternitas/sidekiq"
+require "aeternitas/metric"
 require "aeternitas/metrics"
+require "aeternitas/maintenance"
+require "aeternitas/unique_job_lock"
+require "aeternitas/guard_lock"
+require "aeternitas/aeternitas_job"
+require "aeternitas/poll_job"
+require "aeternitas/cleanup_stale_locks_job"
+require "aeternitas/cleanup_old_metrics_job"
 
 # Aeternitas
 module Aeternitas
-
-  # Get the configured redis connection
-  # @return [ConnectionPool::Wrapper] returns a redis connection from the pool
-  def self.redis
-    @redis ||= ConnectionPool::Wrapper.new(size: 5, timeout: 3) { Redis.new(self.config.redis) }
-  end
-
   # Access the configuration
   # @return [Aeternitas::Configuration] the Aeternitas configuration
   def self.config
@@ -33,36 +30,42 @@ module Aeternitas
   # @see Aeternitas::Configuration
   # @yieldparam [Aeternitas::Configuration] config the aeternitas configuration
   def self.configure
-    yield(self.config)
+    yield(config)
   end
 
   # Enqueues all active pollables for which next polling is lower than the current time
   def self.enqueue_due_pollables
     Aeternitas::PollableMetaData.due.find_each do |pollable_meta_data|
-      Aeternitas::Sidekiq::PollJob
+      Aeternitas::PollJob
         .set(queue: pollable_meta_data.pollable.pollable_configuration.queue)
-        .perform_async(pollable_meta_data.id)
+        .perform_later(pollable_meta_data.id)
       pollable_meta_data.enqueue
       pollable_meta_data.save
     end
   end
 
   # Stores the global Aeternitas configuration
-  # @!attribute [rw] redis
-  #   Redis configuration hash, Default: nil
   # @!attribute [rw] storage_adapter_config
   #   Storage adapter configuration, See {Aeternitas::StorageAdapter} for configuration options
   # @!attribute [rw] storage_adapter
   #   Storage adapter class. Default: {Aeternitas::StorageAdapter::File}
+  # @!attribute [rw] metrics_enabled
+  #   Whether to log metrics to the database. Default: false
+  # @!attribute [rw] metric_retention_period
+  #   How long to keep metric data before it can be cleaned up. Default: 90.days
   class Configuration
-    attr_accessor :redis, :storage_adapter, :storage_adapter_config
+    attr_accessor :storage_adapter,
+      :storage_adapter_config,
+      :metrics_enabled,
+      :metric_retention_period
 
     def initialize
-      @redis = nil
       @storage_adapter = Aeternitas::StorageAdapter::File
       @storage_adapter_config = {
-        directory: defined?(Rails) ? File.join(Rails.root, %w[aeternitas_data]) : File.join(Dir.getwd, 'aeternitas_data')
+        directory: defined?(Rails) ? Rails.root.join("storage", "aeternitas") : File.join(Dir.getwd, "aeternitas_data")
       }
+      @metrics_enabled = false
+      @metric_retention_period = 90.days
     end
 
     # Creates a new StorageAdapter instance with the given options
@@ -70,11 +73,5 @@ module Aeternitas
     def get_storage_adapter
       @storage_adapter.new(storage_adapter_config)
     end
-
-    def redis=(redis_config)
-      @redis = redis_config
-      TabsTabs.configure { |tabstabs_config| tabstabs_config.redis = redis_config }
-    end
   end
 end
-

data/lib/generators/aeternitas/install_generator.rb

@@ -8,28 +8,34 @@ module Aeternitas
 
     source_root File.expand_path("../templates", __FILE__)
 
-    desc 'Generates (but does not run) a migration to add all tables needed by Aeternitas.' \
-         '  Also generates an initializer file for configuring Aeternitas'
+    desc "Generates (but does not run) a migration to add all tables needed by Aeternitas." \
+         "  Also generates an initializer file for configuring Aeternitas"
 
     def create_migration_file
       migration_dir = File.expand_path("db/migrate")
-      if self.class.migration_exists?(migration_dir, 'add_aeternitas')
-        ::Kernel.warn "Migration already exists: #{template}"
+      if self.class.migration_exists?(migration_dir, "add_aeternitas")
+        ::Kernel.warn "Migration 'add_aeternitas' already exists. Skipping."
       else
-        migration_template('add_aeternitas.rb.erb', 'db/migrate/add_aeternitas.rb')
+        migration_template("add_aeternitas.rb.erb", "db/migrate/add_aeternitas.rb")
       end
     end
 
     def copy_initializer
-      copy_file('initializer.rb', 'config/initializers/aeternitas.rb')
+      copy_file("initializer.rb", "config/initializers/aeternitas.rb")
     end
 
     def reminder
-      say "Don't forget to regularly run 'Aeternitas.enqueue_due_pollables'. E.g using 'whenever'", :red
+      say "\nDon't forget to regularly run 'Aeternitas.enqueue_due_pollables', e.g., using 'whenever'", :red
+      say "You should also schedule maintenance jobs:\n", :yellow
+      say "To clean up old metrics (if metrics are enabled):\n"
+      say "    Aeternitas::CleanupOldMetricsJob.perform_later\n", :white
+      say "To clean up stale locks from crashed workers:\n"
+      say "    Aeternitas::CleanupStaleLocksJob.perform_later\n", :white
+      say "Schedule these to run periodically, for example, once a week.\n"
     end
 
     def self.next_migration_number(dirname)
       ::ActiveRecord::Generators::Base.next_migration_number(dirname)
     end
   end
-end
+end

data/lib/generators/aeternitas/templates/add_aeternitas.rb.erb

@@ -1,11 +1,11 @@
 # This migration creates the tables needed by Aeternitas
-class AddAeternitas < ActiveRecord::Migration
+class AddAeternitas < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
     create_table :aeternitas_pollable_meta_data do |t|
       t.string :pollable_type, null: false
       t.integer :pollable_id, null: false
       t.string :pollable_class, null: false
-      t.datetime :next_polling, null: false, default: "1970-01-01 00:00:00+002"
+      t.datetime :next_polling, null: false, default: "1970-01-01 00:00:00"
       t.datetime :last_polling
       t.string :state
       t.text :deactivation_reason
@@ -21,5 +21,37 @@ class AddAeternitas < ActiveRecord::Migration
       t.datetime :created_at
     end
     add_index :aeternitas_sources, [:pollable_id, :pollable_type], name: 'aeternitas_pollable_source'
+
+    create_table :aeternitas_unique_job_locks do |t|
+      t.string :lock_digest, null: false
+      t.string :guard_key_digest
+      t.string :job_id
+      t.datetime :expires_at, null: false
+
+      t.timestamps
+    end
+    add_index :aeternitas_unique_job_locks, :lock_digest, unique: true
+    add_index :aeternitas_unique_job_locks, :guard_key_digest
+    add_index :aeternitas_unique_job_locks, :expires_at
+
+    create_table :aeternitas_guard_locks do |t|
+      t.string :lock_key, null: false
+      t.string :state, null: false
+      t.string :token, null: false, limit: 20
+      t.datetime :locked_until, null: false
+      t.text :reason
+
+      t.timestamps
+    end
+    add_index :aeternitas_guard_locks, :lock_key, unique: true
+    add_index :aeternitas_guard_locks, :locked_until
+
+    create_table :aeternitas_metrics do |t|
+      t.string :name, null: false
+      t.string :pollable_class, null: false
+      t.float :value, null: false
+      t.datetime :created_at, null: false
+    end
+    add_index :aeternitas_metrics, [:name, :pollable_class, :created_at], name: 'idx_aeternitas_metrics'
   end
 end