aeternitas 0.2.0 → 2.0.0.rc1

data/lib/generators/aeternitas/templates/initializer.rb

@@ -1,10 +1,13 @@
-# Configure Aeternitas
+# Aeternitas is configured with sensible defaults. You can override them here if needed.
 Aeternitas.configure do |config|
-  config.redis = { host: 'localhost', port: 6379 } #this is the default Redis config which should work in most cases.
-end
+  # Configure source storage (default is Rails.root.join("storage", "aeternitas"))
+  # config.storage_adapter_config = {
+  #   directory: File.join(Rails.root, 'public', 'sources')
+  # }
+
+  # Configure metrics collection (default is disabled)
+  # config.metrics_enabled = true
 
-Sidekiq.configure_server do |config|
-  config.server_middleware do |chain|
-    chain.add Aeternitas::Sidekiq::Middleware
-  end
+  # Change how long metric data is kept (default is 90 days)
+  # config.metric_retention_period = 180.days
 end

metadata

@@ -1,16 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: aeternitas
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 2.0.0.rc1
 platform: ruby
 authors:
 - Michael Prilop
 - Max Kießling
-- Robert Terbach
-autorequire: 
+- Louis Franzke
 bindir: exe
 cert_chain: []
-date: 2020-07-08 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -18,42 +17,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
-  name: redis
+  name: activejob
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: connection_pool
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: aasm
   requirement: !ruby/object:Gem::Requirement
@@ -68,54 +53,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">"
-      - !ruby/object:Gem::Version
-        version: '4'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '6'
-- !ruby/object:Gem::Dependency
-  name: sidekiq-unique-jobs
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: tabstabs
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -145,63 +82,49 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '7.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
-- !ruby/object:Gem::Dependency
-  name: rspec-sidekiq
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: mock_redis
+  name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -214,27 +137,19 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: |2
-      Æternitas provides means to regularly 'poll' resources (i.e. a website, twitter feed or API) and to permanently
-      store retrieved results. By default æternitas avoids putting too much load on external servers and stores raw
-      results as compressed files on disk. It can be configured to a wide variety of polling strategies (e.g. frequencies,
-      cooldown periods, ignoring exceptions, deactivating resources, ...)."
+description: æternitas is a tool support polling resources (webpages, APIs).
 email:
-- max@kopfueber.org
-- michael.prilop@imw.fraunhofer.de
+- prilop@infai.org
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitattributes"
+- ".github/workflows/lint.yml"
+- ".github/workflows/tests.yml"
 - ".gitignore"
-- ".idea/.rakeTasks"
-- ".idea/misc.xml"
-- ".idea/modules.xml"
-- ".idea/vcs.xml"
-- ".rspec"
-- ".rubocop.yml"
 - ".ruby-version"
-- ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -244,35 +159,33 @@ files:
 - bin/console
 - bin/setup
 - lib/aeternitas.rb
+- lib/aeternitas/aeternitas_job.rb
+- lib/aeternitas/cleanup_old_metrics_job.rb
+- lib/aeternitas/cleanup_stale_locks_job.rb
 - lib/aeternitas/errors.rb
 - lib/aeternitas/guard.rb
+- lib/aeternitas/guard_lock.rb
+- lib/aeternitas/maintenance.rb
+- lib/aeternitas/metric.rb
 - lib/aeternitas/metrics.rb
-- lib/aeternitas/metrics/counter.rb
-- lib/aeternitas/metrics/ratio.rb
-- lib/aeternitas/metrics/ten_minutes_resolution.rb
-- lib/aeternitas/metrics/values.rb
+- lib/aeternitas/poll_job.rb
 - lib/aeternitas/pollable.rb
 - lib/aeternitas/pollable/configuration.rb
 - lib/aeternitas/pollable/dsl.rb
 - lib/aeternitas/pollable_meta_data.rb
 - lib/aeternitas/polling_frequency.rb
-- lib/aeternitas/sidekiq.rb
-- lib/aeternitas/sidekiq/middleware.rb
-- lib/aeternitas/sidekiq/poll_job.rb
 - lib/aeternitas/source.rb
 - lib/aeternitas/storage_adapter.rb
 - lib/aeternitas/storage_adapter/file.rb
+- lib/aeternitas/unique_job_lock.rb
 - lib/aeternitas/version.rb
 - lib/generators/aeternitas/install_generator.rb
 - lib/generators/aeternitas/templates/add_aeternitas.rb.erb
 - lib/generators/aeternitas/templates/initializer.rb
-- logo.png
-- logo.svg
-homepage: https://github.com/FHG-IMW/aeternitas
+homepage: https://github.com/Dietech-Group/aeternitas
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -287,8 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key: 
+rubygems_version: 3.6.9
 specification_version: 4
-summary: æternitas - A ruby gem for continuous source retrieval and data integration
+summary: æternitas - version 2
 test_files: []

data/.idea/.rakeTasks

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<Settings><!--This file was automatically generated by Ruby plugin.
-You are allowed to: 
-1. Remove rake task
-2. Add existing rake tasks
-To add existing rake tasks automatically delete this file and reload the project.
---><RakeGroup description="" fullCmd="" taksId="rake"><RakeTask description="Build aeternitas-0.1.0.gem into the pkg directory" fullCmd="build" taksId="build" /><RakeTask description="Remove any temporary products" fullCmd="clean" taksId="clean" /><RakeTask description="Remove any generated files" fullCmd="clobber" taksId="clobber" /><RakeTask description="Build and install aeternitas-0.1.0.gem into system gems" fullCmd="install" taksId="install" /><RakeGroup description="" fullCmd="" taksId="install"><RakeTask description="Build and install aeternitas-0.1.0.gem into system gems without network access" fullCmd="install:local" taksId="local" /></RakeGroup><RakeTask description="Create tag v0.1.0 and build and push aeternitas-0.1.0.gem to Rubygems" fullCmd="release[remote]" taksId="release[remote]" /><RakeTask description="Run RSpec code examples" fullCmd="spec" taksId="spec" /><RakeTask description="" fullCmd="default" taksId="default" /><RakeTask description="" fullCmd="release" taksId="release" /><RakeGroup description="" fullCmd="" taksId="release"><RakeTask description="" fullCmd="release:guard_clean" taksId="guard_clean" /><RakeTask description="" fullCmd="release:rubygem_push" taksId="rubygem_push" /><RakeTask description="" fullCmd="release:source_control_push" taksId="source_control_push" /></RakeGroup></RakeGroup></Settings>

data/.idea/misc.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="rbenv: 2.4.0" project-jdk-type="RUBY_SDK" />
-</project>

data/.idea/modules.xml

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/aeternitas.iml" filepath="$PROJECT_DIR$/.idea/aeternitas.iml" />
-    </modules>
-  </component>
-</project>

data/.idea/vcs.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$" vcs="Git" />
-  </component>
-</project>

data/.rspec

@@ -1,2 +0,0 @@
---format documentation
---color

data/.rubocop.yml

@@ -1,2 +0,0 @@
-Metrics/LineLength:
-  Max: 100

data/.travis.yml

@@ -1,8 +0,0 @@
-sudo: false
-services:
-  - redis-server
-language: ruby
-rvm:
-  - 2.4.0
-  - 2.5.7
-before_install: gem install bundler -v 1.13.7

data/lib/aeternitas/metrics/counter.rb

@@ -1,18 +0,0 @@
-module Aeternitas
-  module Metrics
-    # Wrapper for {TabsTabs::Metrics::Counter::Stats}.
-    # It is for Counter metrics.
-    class Counter
-      include Enumerable
-      extend Forwardable
-
-      def_delegators :@tabstabs_stats, :min, :max, :avg, :each, :to_a, :first, :last
-
-      # Create a new Wrapper
-      # @param [TabsTabs::Metrics::Counter::Stats] tabstabs_stats the wrapped stats.
-      def initialize(tabstabs_stats)
-        @tabstabs_stats = tabstabs_stats
-      end
-    end
-  end
-end

data/lib/aeternitas/metrics/ratio.rb

@@ -1,67 +0,0 @@
-module Aeternitas
-  module Metrics
-    # Stores time series data for ratios
-    # @!attribute [r] from
-    #   @return [DateTime] start of the time series
-    # @!attribute [r] to
-    #   @return [DateTime] end of the time series
-    # @!attribute [r] resolution
-    #   @return [Symbol] resolution of the time series
-    # @!attribute [r] values
-    #   @return [Array] time series data
-    # The time series values have the following format:
-    #   {
-    #     timestamp: DateTime("2000-01-01 00:00:00 UTC"),
-    #     ratio: 0.01
-    #   }
-    class Ratio
-      include Enumerable
-
-      attr_reader :from, :to, :resolution, :values
-
-      # Create a new ratio time series
-      # @param [DateTime] from start of the time series
-      # @param [DateTime] to end of the time series
-      # @param [Symbol] resolution time series resolution
-      # @param [Array] values time series data
-      def initialize(from, to, resolution, values)
-        @from = from
-        @to = to
-        @resolution = resolution
-        @values = values
-      end
-
-      def each(&block)
-        @values.each(&block)
-      end
-
-      def to_a
-        @values.to_a
-      end
-
-      # Computes the minimum ration within the time series.
-      # @return [Float] the minimum ratio
-      def min
-        @values.min_by { |v| v['ratio'] }['ratio']
-      end
-
-      # Computes the maximum ration within the time series.
-      # @return [Float] the maximum ratio
-      def max
-        @values.max_by { |v| v['ratio'] }['ratio']
-      end
-
-      # Computes the average ration within the time series.
-      # @return [Float] the average ratio
-      def avg
-        return 0 if count.zero?
-        p @values
-        @values.inject(0) { |sum, v| sum + v[:ratio] } / @values.count
-      end
-
-      def to_s
-        values.to_s
-      end
-    end
-  end
-end

data/lib/aeternitas/metrics/ten_minutes_resolution.rb

@@ -1,40 +0,0 @@
-module Aeternitas
-  module Metrics
-    # A TabsTabs resolution represeting 10 minute intervals.
-    module TenMinutesResolution
-      include TabsTabs::Resolutionable
-      extend self
-
-      PATTERN = '%Y-%m-%d-%H-%M'.freeze
-
-      def name
-        :ten_minutes
-      end
-
-      def serialize(ts)
-        Time.utc(ts.year, ts.month, ts.day, ts.hour, (ts.min / 10).to_i).strftime(PATTERN)
-      end
-
-      def deserialize(str)
-        dt = DateTime.strptime(str, PATTERN)
-        normalize(dt)
-      end
-
-      def from_seconds(s)
-        s / 10.minutes
-      end
-
-      def to_seconds
-        10.minutes
-      end
-
-      def add(ts, number)
-        ts + number * 10.minutes
-      end
-
-      def normalize(ts)
-        Time.utc(ts.year, ts.month, ts.day, ts.hour, ts.min)
-      end
-    end
-  end
-end

data/lib/aeternitas/metrics/values.rb

@@ -1,18 +0,0 @@
-module Aeternitas
-  module Metrics
-    # Wrapper for {TabsTabs::Metrics::Value::Stats}.
-    # It is for Value metrics.
-    class Values
-      include Enumerable
-      extend Forwardable
-
-      def_delegators :@tabstabs_stats, :sum, :min, :max, :avg, :each, :to_a, :first, :last
-
-      # Create a new Wrapper
-      # @param [TabsTabs::Metrics::Value::Stats] tabstabs_stats the wrapped stats.
-      def initialize(tabstabs_stats)
-        @tabstabs_stats = tabstabs_stats
-      end
-    end
-  end
-end

data/lib/aeternitas/sidekiq/middleware.rb

@@ -1,31 +0,0 @@
-module Aeternitas
-  module Sidekiq
-    # Aeternitas Sidekiq Middleware
-    class Middleware
-
-      def call(worker, msg, queue)
-        yield
-      rescue Aeternitas::Guard::GuardIsLocked => e
-        raise e unless worker.is_a? Aeternitas::Sidekiq::PollJob
-
-        # try deleting the sidekiq unique key so the job can be reenqueued
-        Aeternitas.redis.del(
-          SidekiqUniqueJobs::UniqueArgs.digest(msg)
-        )
-
-        # reenqueue the job
-        worker.class.client_push msg
-
-        # update the pollables state
-        meta_data = Aeternitas::PollableMetaData.find_by(id: msg['args'].first)
-        meta_data.enqueue!
-
-        if meta_data.pollable.pollable_configuration.sleep_on_guard_locked
-          # put the worker to rest for errors timeout
-          sleep_duration = (e.timeout - Time.now).to_i
-          sleep(sleep_duration + 1.0) if sleep_duration > 0
-        end
-      end
-    end
-  end
-end

data/lib/aeternitas/sidekiq/poll_job.rb

@@ -1,30 +0,0 @@
-require 'sidekiq'
-module Aeternitas
-  module Sidekiq
-    # Sidekiq Worker that is responsible for executing the polling.
-    class PollJob
-      include ::Sidekiq::Worker
-
-      sidekiq_options unique: :until_executed,
-                      unique_args: [:pollable_meta_data_id],
-                      unique_job_expiration: 1.month.to_i,
-                      queue: :polling,
-                      retry: 4
-
-      sidekiq_retry_in do |count|
-        [60, 3600, 86400, 604800][count]
-      end
-
-      sidekiq_retries_exhausted do |msg|
-        meta_data = Aeternitas::PollableMetaData.find_by!(id: msg['args'].first)
-        meta_data.disable_polling(msg['error_message'])
-      end
-
-      def perform(pollable_meta_data_id)
-        meta_data = Aeternitas::PollableMetaData.find_by(id: pollable_meta_data_id)
-        pollable = meta_data.pollable
-        pollable.execute_poll
-      end
-    end
-  end
-end

data/lib/aeternitas/sidekiq.rb

@@ -1,5 +0,0 @@
-require 'aeternitas/sidekiq/poll_job'
-require 'aeternitas/sidekiq/middleware'
-module Aeternitas
-  module Sidekiq ; end
-end