aeternitas 0.1.0

data/lib/aeternitas/metrics/counter.rb

@@ -0,0 +1,18 @@
+module Aeternitas
+  module Metrics
+    # Wrapper for {TabsTabs::Metrics::Counter::Stats}.
+    # It is for Counter metrics.
+    class Counter
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@tabstabs_stats, :min, :max, :avg, :each, :to_a, :first, :last
+
+      # Create a new Wrapper
+      # @param [TabsTabs::Metrics::Counter::Stats] tabstabs_stats the wrapped stats.
+      def initialize(tabstabs_stats)
+        @tabstabs_stats = tabstabs_stats
+      end
+    end
+  end
+end

data/lib/aeternitas/metrics/ratio.rb

@@ -0,0 +1,67 @@
+module Aeternitas
+  module Metrics
+    # Stores time series data for ratios
+    # @!attribute [r] from
+    #   @return [DateTime] start of the time series
+    # @!attribute [r] to
+    #   @return [DateTime] end of the time series
+    # @!attribute [r] resolution
+    #   @return [Symbol] resolution of the time series
+    # @!attribute [r] values
+    #   @return [Array] time series data
+    # The time series values have the following format:
+    #   {
+    #     timestamp: DateTime("2000-01-01 00:00:00 UTC"),
+    #     ratio: 0.01
+    #   }
+    class Ratio
+      include Enumerable
+
+      attr_reader :from, :to, :resolution, :values
+
+      # Create a new ratio time series
+      # @param [DateTime] from start of the time series
+      # @param [DateTime] to end of the time series
+      # @param [Symbol] resolution time series resolution
+      # @param [Array] values time series data
+      def initialize(from, to, resolution, values)
+        @from = from
+        @to = to
+        @resolution = resolution
+        @values = values
+      end
+
+      def each(&block)
+        @values.each(&block)
+      end
+
+      def to_a
+        @values.to_a
+      end
+
+      # Computes the minimum ration within the time series.
+      # @return [Float] the minimum ratio
+      def min
+        @values.min_by { |v| v['ratio'] }['ratio']
+      end
+
+      # Computes the maximum ration within the time series.
+      # @return [Float] the maximum ratio
+      def max
+        @values.max_by { |v| v['ratio'] }['ratio']
+      end
+
+      # Computes the average ration within the time series.
+      # @return [Float] the average ratio
+      def avg
+        return 0 if count.zero?
+        p @values
+        @values.inject(0) { |sum, v| sum + v[:ratio] } / @values.count
+      end
+
+      def to_s
+        values.to_s
+      end
+    end
+  end
+end

data/lib/aeternitas/metrics/ten_minutes_resolution.rb

@@ -0,0 +1,40 @@
+module Aeternitas
+  module Metrics
+    # A TabsTabs resolution represeting 10 minute intervals.
+    module TenMinutesResolution
+      include TabsTabs::Resolutionable
+      extend self
+
+      PATTERN = '%Y-%m-%d-%H-%M'.freeze
+
+      def name
+        :ten_minutes
+      end
+
+      def serialize(ts)
+        Time.utc(ts.year, ts.month, ts.day, ts.hour, (ts.min / 10).to_i).strftime(PATTERN)
+      end
+
+      def deserialize(str)
+        dt = DateTime.strptime(str, PATTERN)
+        normalize(dt)
+      end
+
+      def from_seconds(s)
+        s / 10.minutes
+      end
+
+      def to_seconds
+        10.minutes
+      end
+
+      def add(ts, number)
+        ts + number * 10.minutes
+      end
+
+      def normalize(ts)
+        Time.utc(ts.year, ts.month, ts.day, ts.hour, ts.min)
+      end
+    end
+  end
+end

data/lib/aeternitas/metrics/values.rb

@@ -0,0 +1,18 @@
+module Aeternitas
+  module Metrics
+    # Wrapper for {TabsTabs::Metrics::Value::Stats}.
+    # It is for Value metrics.
+    class Values
+      include Enumerable
+      extend Forwardable
+
+      def_delegators :@tabstabs_stats, :sum, :min, :max, :avg, :each, :to_a, :first, :last
+
+      # Create a new Wrapper
+      # @param [TabsTabs::Metrics::Value::Stats] tabstabs_stats the wrapped stats.
+      def initialize(tabstabs_stats)
+        @tabstabs_stats = tabstabs_stats
+      end
+    end
+  end
+end

data/lib/aeternitas/pollable.rb

@@ -0,0 +1,197 @@
+require 'aeternitas/pollable/configuration'
+require 'aeternitas/pollable/dsl'
+
+module Aeternitas
+  # Mixin that enables the frequent polling of the receiving class.
+  # Classes including this method must implement the .poll method.
+  # Polling behaviour can be configured via {.pollable_options}.
+  # @note Can only be used by classes inheriting from ActiveRecord::Base
+  # @example
+  #   class MyWebsitePollable < ActiveRecord::Base
+  #     includes Aeternitas::Pollable
+  #
+  #     polling_options do
+  #       polling_frequency :daily
+  #       lock_key ->(obj) {obj.url}
+  #     end
+  #
+  #     def poll
+  #       response = HTTParty.get(self.url)
+  #       raise StandardError, "#{self.url} responded with #{response.status}" unless response.success?
+  #       HttpSource.create!(content: response.parsed_response)
+  #     end
+  #   end
+  module Pollable
+    extend ActiveSupport::Concern
+
+    included do
+      raise StandardError, 'Aeternitas::Pollable must inherit from ActiveRecord::Base' unless self.ancestors.include?(ActiveRecord::Base)
+
+      has_one :pollable_meta_data, as: :pollable,
+              dependent: :destroy,
+              class_name: 'Aeternitas::PollableMetaData'
+
+      has_many :sources, as: :pollable,
+               dependent: :destroy,
+               class_name: 'Aeternitas::Source'
+
+      validates :pollable_meta_data, presence: true
+
+      before_validation ->(pollable) do
+        pollable.pollable_meta_data ||= pollable.build_pollable_meta_data(state: 'waiting')
+        pollable.pollable_meta_data.pollable_class = pollable.class.name
+      end
+
+      after_commit ->(pollable) { Aeternitas::Metrics.log(:pollables_created, pollable.class) }, on: :create
+
+      delegate :next_polling, :last_polling, :disable_polling, to: :pollable_meta_data
+    end
+
+    # This method runs the polling workflow
+    def execute_poll
+      _before_poll
+
+      begin
+        guard.with_lock { poll }
+      rescue StandardError => e
+        if pollable_configuration.deactivation_errors.include?(e.class)
+          disable_polling(e)
+          return false
+        elsif pollable_configuration.ignored_errors.include?(e.class)
+          pollable_meta_data.has_errored!
+          raise Aeternitas::Errors::Ignored, e
+        else
+          pollable_meta_data.has_errored!
+          raise e
+        end
+      end
+
+      _after_poll
+    rescue StandardError => e
+      begin
+        log_poll_error(e)
+      ensure
+        raise e
+      end
+    end
+
+
+
+    # This method implements the class specific polling behaviour.
+    # It is only called after the lock was acquired successfully.
+    #
+    # @abstract This method must be implemented when {Aeternitas::Pollable} is included
+    def poll
+      raise NotImplementedError, "#{self.class.name} does not implement #poll, required by Aeternitas::Pollable"
+    end
+
+    # Registers the instance as pollable.
+    #
+    # @note Manual registration is only needed if the object was created before
+    #   {Aeternitas::Pollable} was included. Otherwise it is done automatically after creation.
+    def register_pollable
+      self.pollable_meta_data ||= create_pollable_meta_data(
+          state: 'waiting',
+          pollable_class: self.class.name
+      )
+    end
+
+    def guard
+      guard_key = pollable_configuration.guard_options[:key].call(self)
+      guard_timeout = pollable_configuration.guard_options[:timeout]
+      guard_cooldown = pollable_configuration.guard_options[:cooldown]
+      Aeternitas::Guard.new(guard_key, guard_cooldown, guard_timeout)
+    end
+
+    # Access the Pollables configuration
+    #
+    # @return [Aeternitas::Pollable::Configuration] the pollables configuration
+    def pollable_configuration
+      self.class.pollable_configuration
+    end
+
+    # Creates a new source with the given content if it does not exist
+    # @example
+    #   #...
+    #   def poll
+    #     response = HTTParty.get("http://example.com")
+    #     add_source(response.parsed_response)
+    #   end
+    #   #...
+    # @param [String] raw_content the sources raw content
+    # @return [Aeternitas::Source] the newly created or existing source
+    def add_source(raw_content)
+      source = self.sources.create(raw_content: raw_content)
+      return nil unless source.persisted?
+
+      Aeternitas::Metrics.log(:sources_created, self.class)
+      source
+    end
+
+    private
+
+    # Run all prepolling methods
+    def _before_poll
+      @start_time = Time.now
+      Aeternitas::Metrics.log(:polls, self.class)
+
+      pollable_configuration.before_polling.each { |action| action.call(self) }
+      pollable_meta_data.poll!
+    end
+
+    # Run all postpolling methods
+    def _after_poll
+      pollable_meta_data.wait! do
+        pollable_meta_data.update_attributes!(
+          last_polling: Time.now,
+          next_polling: pollable_configuration.polling_frequency.call(self)
+        )
+      end
+
+      pollable_configuration.after_polling.each { |action| action.call(self) }
+
+      if @start_time
+        execution_time = Time.now - @start_time
+        Aeternitas::Metrics.log_value(:execution_time, self.class, execution_time)
+        Aeternitas::Metrics.log(:guard_timeout_exceeded, self.class) if execution_time > pollable_configuration.guard_options[:timeout]
+        @start_time = nil
+      end
+      Aeternitas::Metrics.log(:successful_polls, self.class)
+    end
+
+    def log_poll_error(e)
+      if e.is_a? Aeternitas::Guard::GuardIsLocked
+        Aeternitas::Metrics.log(:guard_locked, self.class)
+        Aeternitas::Metrics.log_value(:guard_timeout, self.class, e.timeout - Time.now)
+      elsif e.is_a? Aeternitas::Errors::Ignored
+        Aeternitas::Metrics.log(:ignored_error, self.class)
+        Aeternitas::Metrics.log(:failed_polls, self.class)
+      else
+        Aeternitas::Metrics.log(:failed_polls, self.class)
+      end
+    end
+
+    class_methods do
+      # Access the Pollables configuration
+      # @return [Aeternitas::Pollable::Configuration] the pollables configuration
+      def pollable_configuration
+        @pollable_configuration ||= Aeternitas::Pollable::Configuration.new
+      end
+
+      def pollable_configuration=(config)
+        @pollable_configuration = config
+      end
+
+      # Configure the polling process.
+      # For available configuration options see {Aeternitas::Pollable::Configuration} and {Aeternitas::Pollable::DSL}
+      def polling_options(&block)
+        Aeternitas::Pollable::Dsl.new(self.pollable_configuration, &block)
+      end
+
+      def inherited(other)
+        super
+        other.pollable_configuration = @pollable_configuration.copy
+      end
+    end
+  end
+end

data/lib/aeternitas/pollable/configuration.rb

@@ -0,0 +1,64 @@
+module Aeternitas
+  module Pollable
+    # Holds the configuration of a pollable class
+    # @!attribute [rw] polling_frequency
+    #   Method that calculates the next polling time after a successful poll.
+    #   (Default: {Aeternitas::PollingFrequency::DAILY})
+    # @!attribute [rw] before_polling
+    #   Methods to be run before each poll
+    # @!attribute [rw] after_polling
+    #   Methods to be run after each successful poll
+    # @!attribute [rw] queue
+    #   Sidekiq queue the poll job will be enqueued in (Default: 'polling')
+    # @!attribute [rw] guard_options
+    #   Configuration of the pollables lock (Default: key => class+id, cooldown => 5.seconds, timeout => 10.minutes)
+    # @!attribute [rw] deactivation_errors
+    #   The pollable instance will be deactivated if any of these errors occur while polling
+    # @!attribute [rw] ignored_errors
+    #   Errors in this list will be wrapped by {Aeternitas::Error::Ignored} if they occur while polling
+    #   (i.e. ignore in your exception tracker)
+    # @!attribute [rw] sleep_on_guard_locked
+    #   When set to true poll jobs (and effectively the Sidekiq worker thread) will sleep until the
+    #   lock is released if the lock could not be acquired. (Default: true)
+    class Configuration
+      attr_accessor :deactivation_errors,
+                    :before_polling,
+                    :queue,
+                    :polling_frequency,
+                    :after_polling,
+                    :guard_options,
+                    :ignored_errors,
+                    :sleep_on_guard_locked
+
+      # Creates a new Configuration with default options
+      def initialize
+        @polling_frequency = Aeternitas::PollingFrequency::DAILY
+        @before_polling = []
+        @after_polling = []
+        @guard_options = {
+          key: ->(obj) { return obj.class.name.to_s },
+          timeout: 10.minutes,
+          cooldown: 5.seconds
+        }
+        @deactivation_errors = []
+        @ignored_errors = []
+        @queue = 'polling'
+        @sleep_on_guard_locked = true
+      end
+
+      def copy
+        config = Configuration.new
+        config.polling_frequency = self.polling_frequency
+        config.before_polling = self.before_polling.deep_dup
+        config.after_polling = self.after_polling.deep_dup
+        config.guard_options = self.guard_options.deep_dup
+        config.deactivation_errors = self.deactivation_errors.deep_dup
+        config.ignored_errors = self.ignored_errors.deep_dup
+        config.queue = self.queue
+        config.sleep_on_guard_locked = self.sleep_on_guard_locked
+        config
+      end
+    end
+  end
+end
+

data/lib/aeternitas/pollable/dsl.rb

@@ -0,0 +1,124 @@
+module Aeternitas
+  module Pollable
+    # DSL wrapper to conveniently configure pollables
+    class Dsl
+      # Create a new DSL instance and configure the configuration with the given block
+      # @param [Aeternitas::Pollable::Configuration] configuration a pollables configuration
+      # @param [Proc] block configuration block
+      def initialize(configuration, &block)
+        @configuration = configuration
+        instance_eval(&block)
+      end
+
+      # Configures the polling frequency. This can be either the name of a {Aeternitas::PollingFrequency}
+      # or a lambda that receives a pollable instance and returns a DateTime
+      #
+      # @param [Symbol, Proc] frequency Sets the polling frequency.
+      #   representing the next polling time.
+      # @example using a preset
+      #   polling_frequency :weekly
+      # @example using a custom block
+      #   polling_frequency ->(pollable) {Time.now + 1.month + Time.now - pollable.created_at.to_i / 3.month * 1.month}
+      # @todo allow custom methods via reference
+      def polling_frequency(frequency)
+        if frequency.is_a?(Symbol)
+          @configuration.polling_frequency = Aeternitas::PollingFrequency.by_name(frequency)
+        else
+          @configuration.polling_frequency = frequency
+        end
+      end
+
+      # Configures a method that will be run before every poll
+      #
+      # @param [Symbol, Block] method method or method name
+      # @example method by reference
+      #   before_polling :my_method
+      #   ...
+      #   def my_method(pollable) do_something end
+      # @example method by block
+      #   before_polling ->(pollable) {do_something}
+      def before_polling(method)
+        if method.is_a?(Symbol)
+          @configuration.before_polling << ->(pollable) { pollable.send(method) }
+        else
+          @configuration.before_polling << method
+        end
+      end
+
+      # Configures a method that will be run after every successful poll
+      #
+      # @param [Symbol, Block] method method or method name
+      # @example method by reference
+      #   after:polling :my_method
+      #   ...
+      #   def my_method(pollable) do_something end
+      # @example method by block
+      #   after_polling ->(pollable) {do_something}
+      def after_polling(method)
+        if method.is_a?(Symbol)
+          @configuration.after_polling << ->(pollable) { pollable.send(method) }
+        else
+          @configuration.after_polling << method
+        end
+      end
+
+      # Configure errors that will cause the pollable instance to be deactivated immediately during poll.
+      #
+      # @param [Object] error_class error classes
+      def deactivate_on(*error_class)
+        @configuration.deactivation_errors |= error_class
+      end
+
+      # Configure errors that will be wrapped in {Aeternitas::Error::Ignored}.
+      # Use this to group exceptions which should be ignored in your exception tracker.
+      #
+      # @param [Object] error_class error classes
+      def ignore_error(*error_class)
+        @configuration.ignored_errors |= error_class
+      end
+
+      # Configure the Sidekiq queue into which the instance's poll jobs will be enqueued.
+      # @param [String] queue name of the Sidekiq queue
+      def queue(queue)
+        @configuration.queue = queue
+      end
+
+      # Configure the guard key. This can be either a fixed String, a method reference or a block
+      #
+      # @param [String, Symbol, Proc] key lock key
+      # @example using a fixed String
+      #   guard_key "MyLockKey"
+      # @example using a method reference
+      #   guard_key :url
+      # @example using a block
+      #   guard_key ->(pollable) {URI.parse(pollable.url).host}
+      def guard_key(key)
+        @configuration.guard_options[:key] = case key
+                                when Symbol
+                                  ->(obj) { return obj.send(key) }
+                                when Proc
+                                  key
+                                else
+                                  ->(obj) { return key.to_s }
+                                end
+      end
+
+      # Configure the guard.
+      # @see guard_key
+      # @see Aeternitas::Guard
+      # @param [Hash] options guard options
+      def guard_options(options)
+        @configuration.guard_options.merge!(options)
+      end
+
+      # Configure the behaviour of poll jobs if a lock can't be acquired.
+      # When set to true poll jobs (and effectively the Sidekiq worker thread) will sleep until the
+      # lock is released if the lock could not be acquired.
+      # @param [Boolean] switch true|false
+      def sleep_on_guard_locked(switch)
+        @configuration.sleep_on_guard_locked = switch
+      end
+    end
+  end
+end
+