aeternitas 0.2.0 → 2.0.0

data/lib/aeternitas.rb

@@ -1,8 +1,5 @@
+require "ostruct"
 require "active_support/all"
-require "redis"
-require "connection_pool"
-require "sidekiq-unique-jobs"
-require "tabs_tabs"
 require "aeternitas/version"
 require "aeternitas/guard"
 require "aeternitas/pollable"
@@ -11,18 +8,19 @@ require "aeternitas/source"
 require "aeternitas/polling_frequency"
 require "aeternitas/errors"
 require "aeternitas/storage_adapter"
-require "aeternitas/sidekiq"
+require "aeternitas/metric"
 require "aeternitas/metrics"
+require "aeternitas/maintenance"
+require "aeternitas/unique_job_lock"
+require "aeternitas/guard_lock"
+require "aeternitas/aeternitas_job"
+require "aeternitas/poll_job"
+require "aeternitas/cleanup_stale_locks_job"
+require "aeternitas/cleanup_old_metrics_job"
+require "aeternitas/test"
 
 # Aeternitas
 module Aeternitas
-
-  # Get the configured redis connection
-  # @return [ConnectionPool::Wrapper] returns a redis connection from the pool
-  def self.redis
-    @redis ||= ConnectionPool::Wrapper.new(size: 5, timeout: 3) { Redis.new(self.config.redis) }
-  end
-
   # Access the configuration
   # @return [Aeternitas::Configuration] the Aeternitas configuration
   def self.config
@@ -33,36 +31,54 @@ module Aeternitas
   # @see Aeternitas::Configuration
   # @yieldparam [Aeternitas::Configuration] config the aeternitas configuration
   def self.configure
-    yield(self.config)
+    yield(config)
+  end
+
+  # Returns true if aeternitas is in test mode.
+  # @return [Boolean]
+  def self.test_mode?
+    @test_mode == true
+  end
+
+  # Sets the test mode.
+  # @param [Boolean] value
+  def self.test_mode=(value)
+    @test_mode = value
   end
 
   # Enqueues all active pollables for which next polling is lower than the current time
   def self.enqueue_due_pollables
     Aeternitas::PollableMetaData.due.find_each do |pollable_meta_data|
-      Aeternitas::Sidekiq::PollJob
+      Aeternitas::PollJob
         .set(queue: pollable_meta_data.pollable.pollable_configuration.queue)
-        .perform_async(pollable_meta_data.id)
+        .perform_later(pollable_meta_data.id)
       pollable_meta_data.enqueue
       pollable_meta_data.save
     end
   end
 
   # Stores the global Aeternitas configuration
-  # @!attribute [rw] redis
-  #   Redis configuration hash, Default: nil
   # @!attribute [rw] storage_adapter_config
   #   Storage adapter configuration, See {Aeternitas::StorageAdapter} for configuration options
   # @!attribute [rw] storage_adapter
   #   Storage adapter class. Default: {Aeternitas::StorageAdapter::File}
+  # @!attribute [rw] metrics_enabled
+  #   Whether to log metrics to the database. Default: false
+  # @!attribute [rw] metric_retention_period
+  #   How long to keep metric data before it can be cleaned up. Default: 90.days
   class Configuration
-    attr_accessor :redis, :storage_adapter, :storage_adapter_config
+    attr_accessor :storage_adapter,
+      :storage_adapter_config,
+      :metrics_enabled,
+      :metric_retention_period
 
     def initialize
-      @redis = nil
       @storage_adapter = Aeternitas::StorageAdapter::File
       @storage_adapter_config = {
-        directory: defined?(Rails) ? File.join(Rails.root, %w[aeternitas_data]) : File.join(Dir.getwd, 'aeternitas_data')
+        directory: defined?(Rails) ? Rails.root.join("storage", "aeternitas") : File.join(Dir.getwd, "aeternitas_data")
       }
+      @metrics_enabled = false
+      @metric_retention_period = 90.days
     end
 
     # Creates a new StorageAdapter instance with the given options
@@ -70,11 +86,5 @@ module Aeternitas
     def get_storage_adapter
       @storage_adapter.new(storage_adapter_config)
     end
-
-    def redis=(redis_config)
-      @redis = redis_config
-      TabsTabs.configure { |tabstabs_config| tabstabs_config.redis = redis_config }
-    end
   end
 end
-

data/lib/generators/aeternitas/install_generator.rb

@@ -8,28 +8,34 @@ module Aeternitas
 
     source_root File.expand_path("../templates", __FILE__)
 
-    desc 'Generates (but does not run) a migration to add all tables needed by Aeternitas.' \
-         '  Also generates an initializer file for configuring Aeternitas'
+    desc "Generates (but does not run) a migration to add all tables needed by Aeternitas." \
+         "  Also generates an initializer file for configuring Aeternitas"
 
     def create_migration_file
       migration_dir = File.expand_path("db/migrate")
-      if self.class.migration_exists?(migration_dir, 'add_aeternitas')
-        ::Kernel.warn "Migration already exists: #{template}"
+      if self.class.migration_exists?(migration_dir, "add_aeternitas")
+        ::Kernel.warn "Migration 'add_aeternitas' already exists. Skipping."
       else
-        migration_template('add_aeternitas.rb.erb', 'db/migrate/add_aeternitas.rb')
+        migration_template("add_aeternitas.rb.erb", "db/migrate/add_aeternitas.rb")
       end
     end
 
     def copy_initializer
-      copy_file('initializer.rb', 'config/initializers/aeternitas.rb')
+      copy_file("initializer.rb", "config/initializers/aeternitas.rb")
     end
 
     def reminder
-      say "Don't forget to regularly run 'Aeternitas.enqueue_due_pollables'. E.g using 'whenever'", :red
+      say "\nDon't forget to regularly run 'Aeternitas.enqueue_due_pollables', e.g., using 'whenever'", :red
+      say "You should also schedule maintenance jobs:\n", :yellow
+      say "To clean up old metrics (if metrics are enabled):\n"
+      say "    Aeternitas::CleanupOldMetricsJob.perform_later\n", :white
+      say "To clean up stale locks from crashed workers:\n"
+      say "    Aeternitas::CleanupStaleLocksJob.perform_later\n", :white
+      say "Schedule these to run periodically, for example, once a week.\n"
     end
 
     def self.next_migration_number(dirname)
       ::ActiveRecord::Generators::Base.next_migration_number(dirname)
     end
   end
-end
+end

data/lib/generators/aeternitas/templates/add_aeternitas.rb.erb

@@ -1,11 +1,11 @@
 # This migration creates the tables needed by Aeternitas
-class AddAeternitas < ActiveRecord::Migration
+class AddAeternitas < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
   def change
     create_table :aeternitas_pollable_meta_data do |t|
       t.string :pollable_type, null: false
       t.integer :pollable_id, null: false
       t.string :pollable_class, null: false
-      t.datetime :next_polling, null: false, default: "1970-01-01 00:00:00+002"
+      t.datetime :next_polling, null: false, default: "1970-01-01 00:00:00"
       t.datetime :last_polling
       t.string :state
       t.text :deactivation_reason
@@ -21,5 +21,37 @@ class AddAeternitas < ActiveRecord::Migration
       t.datetime :created_at
     end
     add_index :aeternitas_sources, [:pollable_id, :pollable_type], name: 'aeternitas_pollable_source'
+
+    create_table :aeternitas_unique_job_locks do |t|
+      t.string :lock_digest, null: false
+      t.string :guard_key_digest
+      t.string :job_id
+      t.datetime :expires_at, null: false
+
+      t.timestamps
+    end
+    add_index :aeternitas_unique_job_locks, :lock_digest, unique: true
+    add_index :aeternitas_unique_job_locks, :guard_key_digest
+    add_index :aeternitas_unique_job_locks, :expires_at
+
+    create_table :aeternitas_guard_locks do |t|
+      t.string :lock_key, null: false
+      t.string :state, null: false
+      t.string :token, null: false, limit: 20
+      t.datetime :locked_until, null: false
+      t.text :reason
+
+      t.timestamps
+    end
+    add_index :aeternitas_guard_locks, :lock_key, unique: true
+    add_index :aeternitas_guard_locks, :locked_until
+
+    create_table :aeternitas_metrics do |t|
+      t.string :name, null: false
+      t.string :pollable_class, null: false
+      t.float :value, null: false
+      t.datetime :created_at, null: false
+    end
+    add_index :aeternitas_metrics, [:name, :pollable_class, :created_at], name: 'idx_aeternitas_metrics'
   end
 end

data/lib/generators/aeternitas/templates/initializer.rb

@@ -1,10 +1,13 @@
-# Configure Aeternitas
+# Aeternitas is configured with sensible defaults. You can override them here if needed.
 Aeternitas.configure do |config|
-  config.redis = { host: 'localhost', port: 6379 } #this is the default Redis config which should work in most cases.
-end
+  # Configure source storage (default is Rails.root.join("storage", "aeternitas"))
+  # config.storage_adapter_config = {
+  #   directory: File.join(Rails.root, 'public', 'sources')
+  # }
+
+  # Configure metrics collection (default is disabled)
+  # config.metrics_enabled = true
 
-Sidekiq.configure_server do |config|
-  config.server_middleware do |chain|
-    chain.add Aeternitas::Sidekiq::Middleware
-  end
+  # Change how long metric data is kept (default is 90 days)
+  # config.metric_retention_period = 180.days
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: aeternitas
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Michael Prilop
 - Max Kießling
-- Robert Terbach
-autorequire: 
+- Louis Franzke
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-07-08 00:00:00.000000000 Z
+date: 2025-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -18,42 +18,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
-  name: redis
+  name: activejob
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: connection_pool
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: aasm
   requirement: !ruby/object:Gem::Requirement
@@ -68,54 +54,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
-- !ruby/object:Gem::Dependency
-  name: sidekiq-unique-jobs
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: tabstabs
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -145,63 +83,49 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '7.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
-- !ruby/object:Gem::Dependency
-  name: rspec-sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: mock_redis
+  name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -214,27 +138,19 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: |2
-      Æternitas provides means to regularly 'poll' resources (i.e. a website, twitter feed or API) and to permanently
-      store retrieved results. By default æternitas avoids putting too much load on external servers and stores raw
-      results as compressed files on disk. It can be configured to a wide variety of polling strategies (e.g. frequencies,
-      cooldown periods, ignoring exceptions, deactivating resources, ...)."
+description: æternitas is a tool support polling resources (webpages, APIs).
 email:
-- max@kopfueber.org
-- michael.prilop@imw.fraunhofer.de
+- prilop@infai.org
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitattributes"
+- ".github/workflows/lint.yml"
+- ".github/workflows/tests.yml"
 - ".gitignore"
-- ".idea/.rakeTasks"
-- ".idea/misc.xml"
-- ".idea/modules.xml"
-- ".idea/vcs.xml"
-- ".rspec"
-- ".rubocop.yml"
 - ".ruby-version"
-- ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -244,35 +160,35 @@ files:
 - bin/console
 - bin/setup
 - lib/aeternitas.rb
+- lib/aeternitas/aeternitas_job.rb
+- lib/aeternitas/cleanup_old_metrics_job.rb
+- lib/aeternitas/cleanup_stale_locks_job.rb
 - lib/aeternitas/errors.rb
 - lib/aeternitas/guard.rb
+- lib/aeternitas/guard_lock.rb
+- lib/aeternitas/maintenance.rb
+- lib/aeternitas/metric.rb
 - lib/aeternitas/metrics.rb
-- lib/aeternitas/metrics/counter.rb
-- lib/aeternitas/metrics/ratio.rb
-- lib/aeternitas/metrics/ten_minutes_resolution.rb
-- lib/aeternitas/metrics/values.rb
+- lib/aeternitas/poll_job.rb
 - lib/aeternitas/pollable.rb
 - lib/aeternitas/pollable/configuration.rb
 - lib/aeternitas/pollable/dsl.rb
 - lib/aeternitas/pollable_meta_data.rb
 - lib/aeternitas/polling_frequency.rb
-- lib/aeternitas/sidekiq.rb
-- lib/aeternitas/sidekiq/middleware.rb
-- lib/aeternitas/sidekiq/poll_job.rb
 - lib/aeternitas/source.rb
 - lib/aeternitas/storage_adapter.rb
 - lib/aeternitas/storage_adapter/file.rb
+- lib/aeternitas/test.rb
+- lib/aeternitas/unique_job_lock.rb
 - lib/aeternitas/version.rb
 - lib/generators/aeternitas/install_generator.rb
 - lib/generators/aeternitas/templates/add_aeternitas.rb.erb
 - lib/generators/aeternitas/templates/initializer.rb
-- logo.png
-- logo.svg
-homepage: https://github.com/FHG-IMW/aeternitas
+homepage: https://github.com/Dietech-Group/aeternitas
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -287,8 +203,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.4.19
+signing_key:
 specification_version: 4
-summary: æternitas - A ruby gem for continuous source retrieval and data integration
+summary: æternitas - version 2
 test_files: []

data/.idea/.rakeTasks

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Settings><!--This file was automatically generated by Ruby plugin.
-You are allowed to: 
-1. Remove rake task
-2. Add existing rake tasks
-To add existing rake tasks automatically delete this file and reload the project.
---><RakeGroup description="" fullCmd="" taksId="rake"><RakeTask description="Build aeternitas-0.1.0.gem into the pkg directory" fullCmd="build" taksId="build" /><RakeTask description="Remove any temporary products" fullCmd="clean" taksId="clean" /><RakeTask description="Remove any generated files" fullCmd="clobber" taksId="clobber" /><RakeTask description="Build and install aeternitas-0.1.0.gem into system gems" fullCmd="install" taksId="install" /><RakeGroup description="" fullCmd="" taksId="install"><RakeTask description="Build and install aeternitas-0.1.0.gem into system gems without network access" fullCmd="install:local" taksId="local" /></RakeGroup><RakeTask description="Create tag v0.1.0 and build and push aeternitas-0.1.0.gem to Rubygems" fullCmd="release[remote]" taksId="release[remote]" /><RakeTask description="Run RSpec code examples" fullCmd="spec" taksId="spec" /><RakeTask description="" fullCmd="default" taksId="default" /><RakeTask description="" fullCmd="release" taksId="release" /><RakeGroup description="" fullCmd="" taksId="release"><RakeTask description="" fullCmd="release:guard_clean" taksId="guard_clean" /><RakeTask description="" fullCmd="release:rubygem_push" taksId="rubygem_push" /><RakeTask description="" fullCmd="release:source_control_push" taksId="source_control_push" /></RakeGroup></RakeGroup></Settings>

data/.idea/misc.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="rbenv: 2.4.0" project-jdk-type="RUBY_SDK" />
-</project>

data/.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/aeternitas.iml" filepath="$PROJECT_DIR$/.idea/aeternitas.iml" />
-    </modules>
-  </component>
-</project>

data/.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>

data/.rspec

@@ -1,2 +0,0 @@
---format documentation
---color

data/.rubocop.yml

@@ -1,2 +0,0 @@
-Metrics/LineLength:
-  Max: 100

data/.travis.yml

@@ -1,8 +0,0 @@
-sudo: false
-services:
-  - redis-server
-language: ruby
-rvm:
-  - 2.4.0
-  - 2.5.7
-before_install: gem install bundler -v 1.13.7

data/lib/aeternitas/metrics/counter.rb

@@ -1,18 +0,0 @@
-module Aeternitas
-  module Metrics
-    # Wrapper for {TabsTabs::Metrics::Counter::Stats}.
-    # It is for Counter metrics.
-    class Counter
-      include Enumerable
-      extend Forwardable
-
-      def_delegators :@tabstabs_stats, :min, :max, :avg, :each, :to_a, :first, :last
-
-      # Create a new Wrapper
-      # @param [TabsTabs::Metrics::Counter::Stats] tabstabs_stats the wrapped stats.
-      def initialize(tabstabs_stats)
-        @tabstabs_stats = tabstabs_stats
-      end
-    end
-  end
-end

data/lib/aeternitas/metrics/ratio.rb

@@ -1,67 +0,0 @@
-module Aeternitas
-  module Metrics
-    # Stores time series data for ratios
-    # @!attribute [r] from
-    #   @return [DateTime] start of the time series
-    # @!attribute [r] to
-    #   @return [DateTime] end of the time series
-    # @!attribute [r] resolution
-    #   @return [Symbol] resolution of the time series
-    # @!attribute [r] values
-    #   @return [Array] time series data
-    # The time series values have the following format:
-    #   {
-    #     timestamp: DateTime("2000-01-01 00:00:00 UTC"),
-    #     ratio: 0.01
-    #   }
-    class Ratio
-      include Enumerable
-
-      attr_reader :from, :to, :resolution, :values
-
-      # Create a new ratio time series
-      # @param [DateTime] from start of the time series
-      # @param [DateTime] to end of the time series
-      # @param [Symbol] resolution time series resolution
-      # @param [Array] values time series data
-      def initialize(from, to, resolution, values)
-        @from = from
-        @to = to
-        @resolution = resolution
-        @values = values
-      end
-
-      def each(&block)
-        @values.each(&block)
-      end
-
-      def to_a
-        @values.to_a
-      end
-
-      # Computes the minimum ration within the time series.
-      # @return [Float] the minimum ratio
-      def min
-        @values.min_by { |v| v['ratio'] }['ratio']
-      end
-
-      # Computes the maximum ration within the time series.
-      # @return [Float] the maximum ratio
-      def max
-        @values.max_by { |v| v['ratio'] }['ratio']
-      end
-
-      # Computes the average ration within the time series.
-      # @return [Float] the average ratio
-      def avg
-        return 0 if count.zero?
-        p @values
-        @values.inject(0) { |sum, v| sum + v[:ratio] } / @values.count
-      end
-
-      def to_s
-        values.to_s
-      end
-    end
-  end
-end