aeternitas 0.1.0

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "aeternitas"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/aeternitas.rb

@@ -0,0 +1,80 @@
+require "active_support/all"
+require "redis"
+require "connection_pool"
+require "sidekiq-unique-jobs"
+require "tabs_tabs"
+require "aeternitas/version"
+require "aeternitas/guard"
+require "aeternitas/pollable"
+require "aeternitas/pollable_meta_data"
+require "aeternitas/source"
+require "aeternitas/polling_frequency"
+require "aeternitas/errors"
+require "aeternitas/storage_adapter"
+require "aeternitas/sidekiq"
+require "aeternitas/metrics"
+
+# Aeternitas
+module Aeternitas
+
+  # Get the configured redis connection
+  # @return [ConnectionPool::Wrapper] returns a redis connection from the pool
+  def self.redis
+    @redis ||= ConnectionPool::Wrapper.new(size: 5, timeout: 3) { Redis.new(self.config.redis) }
+  end
+
+  # Access the configuration
+  # @return [Aeternitas::Configuration] the Aeternitas configuration
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  # Configure Aeternitas
+  # @see Aeternitas::Configuration
+  # @yieldparam [Aeternitas::Configuration] config the aeternitas configuration
+  def self.configure
+    yield(self.config)
+  end
+
+  # Enqueues all active pollables for which next polling is lower than the current time
+  def self.enqueue_due_pollables
+    Aeternitas::PollableMetaData.due.find_each do |pollable_meta_data|
+      Aeternitas::Sidekiq::PollJob
+        .set(queue: pollable_meta_data.pollable.pollable_configuration.queue)
+        .perform_async(pollable_meta_data.id)
+      pollable_meta_data.enqueue
+      pollable_meta_data.save
+    end
+  end
+
+  # Stores the global Aeternitas configuration
+  # @!attribute [rw] redis
+  #   Redis configuration hash, Default: nil
+  # @!attribute [rw] storage_adapter_config
+  #   Storage adapter configuration, See {Aeternitas::StorageAdapter} for configuration options
+  # @!attribute [rw] storage_adapter
+  #   Storage adapter class. Default: {Aeternitas::StorageAdapter::File}
+  class Configuration
+    attr_accessor :redis, :storage_adapter, :storage_adapter_config
+
+    def initialize
+      @redis = nil
+      @storage_adapter = Aeternitas::StorageAdapter::File
+      @storage_adapter_config = {
+        directory: defined?(Rails) ? File.join(Rails.root, %w[aeternitas_data]) : File.join(Dir.getwd, 'aeternitas_data')
+      }
+    end
+
+    # Creates a new StorageAdapter instance with the given options
+    # @return [Aeternitas::StoragesAdapter] new storage adapter instance
+    def get_storage_adapter
+      @storage_adapter.new(storage_adapter_config)
+    end
+
+    def redis=(redis_config)
+      @redis = redis_config
+      TabsTabs.configure { |tabstabs_config| tabstabs_config.redis = redis_config }
+    end
+  end
+end
+

data/lib/aeternitas/errors.rb

@@ -0,0 +1,48 @@
+module Aeternitas
+  module Errors
+    # Wrapper for ignored errors.
+    # This can be used to conveniently exclude certain errors from e.g. error trackers
+    # @!attribute [r] original error
+    #   the wrapped error
+    class Ignored < StandardError
+      attr_reader :original_error
+
+      # Create a new Instance.
+      #
+      # @param [StandardError] original_error the wrapped error
+      def initialize(original_error)
+        @original_error = original_error
+        super("#{original_error.class} - #{original_error.message}")
+      end
+
+    end
+
+    # Raised when a source data already exists.
+    # @!attribute [r] fingerprint
+    #   the sources fingerprint
+    class SourceDataExists < StandardError
+      attr_reader :fingerprint
+
+      # Create a new Exception
+      # @param [String] fingerprint the sources fingerprint
+      def initialize(fingerprint)
+        @fingerprint = fingerprint
+        super("The source entry with fingerprint '#{fingerprint}' already exists!")
+      end
+    end
+
+    # Raised when a source entry does not exist.
+    # @!attribute [r] fingerprint
+    #   the sources fingerprint
+    class SourceDataNotFound < StandardError
+      attr_reader :fingerprint
+
+      # Create a new Exception
+      # @param [String] fingerprint the sources fingerprint
+      def initialize(fingerprint)
+        @fingerprint = fingerprint
+        super("The source entry with fingerprint '#{fingerprint}' does not exist!")
+      end
+    end
+  end
+end

data/lib/aeternitas/guard.rb

@@ -0,0 +1,199 @@
+require 'active_support/duration'
+require 'securerandom'
+
+module Aeternitas
+  # A distributed lock that can not be acquired after being unlocked for a certain time (cooldown period).
+  # Using Redis key expiration we ensure locks are released even after workers crash after a configurable timout period.
+  #
+  # @example
+  #   guard = Aeternitas::Guard.new("Twitter-MY_API_KEY", 5.seconds)
+  #   begin
+  #     guard.with_lock do
+  #       twitter_client.user_timeline('Darth_Max')
+  #     end
+  #   rescue Twitter::TooManyRequests => e
+  #     guard.sleep_until(e.rate_limit.reset_at)
+  #     raise Aeternitas::Guard::GuardIsLocked(e.rate_limit.reset_at)
+  #   end
+  #
+  # @!attribute [r] id
+  #   @return [String] the guards id
+  # @!attribute [r] timeout
+  #   @return [ActiveSupport::Duration] the locks timeout duration
+  # @!attribute [r] cooldown
+  #   @return [ActiveSupport::Duration] cooldown time, in which the lock can't be acquired after being released
+  # @!attribute [r] token
+  #   @return [String] cryptographic token which ensures we do not lock/unlock a guard held by another process
+  class Guard
+
+    attr_reader :id, :timeout, :cooldown, :token
+
+    # Create a new Guard
+    #
+    # @param [String] id Lock id
+    # @param [ActiveRecord::Duration] cooldown Cooldown time
+    # @param [ActiveRecord::Duration] timeout Lock timeout
+    # @return [Aeternitas::Guard] Creates a new Instance
+    def initialize(id, cooldown, timeout = 10.minutes)
+      @id       = id
+      @cooldown = cooldown
+      @timeout  = timeout
+      @token    = SecureRandom.hex(10)
+    end
+
+    # Runs a given block if the lock can be acquired and releases the lock afterwards.
+    #
+    # @raise [Aeternitas::LockWithCooldown::GuardIsLocked] if the lock can not be acquired
+    # @example
+    #   Guard.new("MyId", 5.seconds, 10.minutes).with_lock { do_request() }
+    def with_lock
+      acquire_lock!
+      begin
+        yield
+      ensure
+        unlock
+      end
+    end
+
+    # Locks the guard until the given time.
+    #
+    # @param [Time] until_time sleep time
+    # @param [String] msg hint why the guard sleeps
+    def sleep_until(until_time, msg = nil)
+      sleep(until_time, msg)
+    end
+
+    # Locks the guard for the given duration.
+    #
+    # @param [ActiveSupport::Duration] duration sleeping duration
+    # @param [String] msg hint why the guard sleeps
+    def sleep_for(duration, msg = nil)
+      raise ArgumentError, 'duration must be an ActiveRecord::Duration' unless duration.is_a?(ActiveSupport::Duration)
+      sleep_until(duration.from_now, msg)
+    end
+
+    private
+
+    # Tries to acquire the lock.
+    #
+    # @example The Redis value looks like this
+    #   {
+    #     id: 'MyId'
+    #     state: 'processing'
+    #     timeout: '3600'
+    #     cooldown: '5'
+    #     locked_until: '2017-01-01 10:10:00'
+    #     token: '1234567890'
+    #   }
+    # @raise [Aeternitas::Guard::GuardIsLocked] if the lock can not be acquired
+    def acquire_lock!
+      payload = {
+        'id' => @id,
+        'state' => 'processing',
+        'timeout' => @timeout,
+        'cooldown' => @cooldown,
+        'locked_until' => @timeout.from_now,
+        'token' => @token
+      }
+
+      has_lock = Aeternitas.redis.set(@id, JSON.unparse(payload), ex: @timeout.to_i, nx: true)
+
+      raise(GuardIsLocked.new(@id, get_timeout)) unless has_lock
+    end
+
+    # Tries to unlock the guard. This starts the cooldown phase.
+    #
+    # @example The Redis value looks like this
+    #   {
+    #     id: 'MyId'
+    #     state: 'cooldown'
+    #     timeout: '3600'
+    #     cooldown: '5'
+    #     locked_until: '2017-01-01 10:00:05'
+    #     token: '1234567890'
+    #   }
+    def unlock
+      return false unless holds_lock?
+
+      payload = {
+          'id' => @id,
+          'state' => 'cooldown',
+          'timeout' => @timeout,
+          'cooldown' => @cooldown,
+          'locked_until' => @cooldown.from_now,
+          'token' => @token
+      }
+
+      Aeternitas.redis.set(@id, JSON.unparse(payload), ex: @cooldown.to_i)
+    end
+
+    # Lets the guard sleep until the given date.
+    # This means that non can acquire the guards lock
+    #
+    # @example The Redis value looks like this
+    #   {
+    #     id: 'MyId'
+    #     state: 'sleeping'
+    #     timeout: '3600'
+    #     cooldown: '5'
+    #     locked_until: '2017-01-01 13:00'
+    #     message: "API Quota Reached"
+    #   }
+    # @todo Should this raise an error if the lock is not owned by this instance?
+    # @param [Time] sleep_timeout for how long will the guard sleep
+    # @param [String] msg hint why the guard sleeps
+    def sleep(sleep_timeout, msg = nil)
+      payload = {
+          'id' => @id,
+          'state' => 'sleeping',
+          'timeout' => @timeout,
+          'cooldown' => @cooldown,
+          'locked_until' => sleep_timeout
+      }
+      payload.merge(message: msg) if msg
+
+      Aeternitas.redis.set(@id, JSON.unparse(payload), ex: (sleep_timeout - Time.now).seconds.to_i)
+    end
+
+    # Checks if this instance holds the lock. This is done by retrieving the value from redis and
+    # comparing the token value. If they match, than the lock is held by this instance.
+    #
+    # @todo Make the check atomic
+    # @return [Boolean] if the lock is held by this instance
+    def holds_lock?
+      payload = get_payload
+      payload['token'] == @token && payload['state'] == 'processing'
+    end
+
+    # Returns the guards current timeout.
+    #
+    # @return [Time] the guards current timeout
+    def get_timeout
+      payload = get_payload
+      payload['state'] == 'processing' ? payload['cooldown'].to_i.seconds.from_now : Time.parse(payload['locked_until'])
+    end
+
+    # Retrieves the locks payload from redis.
+    #
+    # @return [Hash] the locks payload
+    def get_payload
+      value = Aeternitas.redis.get(@id)
+      return {} unless value
+      JSON.parse(value)
+    end
+
+    # Custom error class thrown when the lock can not be acquired
+    # @!attribute [r] timeout
+    #   @return [DateTime] the locks current timeout
+    class GuardIsLocked < StandardError
+      attr_reader :timeout
+
+      def initialize(resource_id, timeout, reason = nil)
+        msg = "Resource '#{resource_id}' is locked until #{timeout}."
+        msg += " Reason: #{reason}" if reason
+        super(msg)
+        @timeout = timeout
+      end
+    end
+  end
+end

data/lib/aeternitas/metrics.rb

@@ -0,0 +1,162 @@
+require 'aeternitas/metrics/ten_minutes_resolution'
+require 'aeternitas/metrics/counter'
+require 'aeternitas/metrics/values'
+require 'aeternitas/metrics/ratio'
+
+module Aeternitas
+  # Provides extensive metrics for Aeternitas.
+  # Every metric is scoped by pollable class.
+  # Available metrics are:
+  #   - polls => Number of polling runs
+  #   - successful_polls => Number of successful polling runs
+  #   - failed_polls => Number of failed polling runs (includes IgnoredErrors,
+  #     excludes deactivation errors and Lock errors)
+  #   - ignored_errors => Number of raised {Aeternitas::Errors::Ignored}
+  #   - deactivation_errors => Number of errors raised which are declared as deactivation_errors
+  #   - execution_time => Job execution time in seconds
+  #   - guard_locked => Number of encountered locked guards
+  #   - guard_timeout => Time until the guard is unlocked in seconds
+  #   - guard_timeout_exceeded => Number of jobs that ran longer than the guards timeout
+  #   - pollables_created => Number of created pollables
+  #   - sources_created => Number of created sources
+  #
+  # Available Resolutions are:
+  #   - :minute (stored for 3 days)
+  #   - :ten_minutes (stored for 14 days)
+  #   - :hour (stored for 2 months)
+  #   - :day (stored indefinitely)
+  #
+  # Every metric can be accessed via a getter method:
+  # @example
+  #   Aeternitas::Metrics.polls MyPollable, from: 3.days.ago, to: Time.now, resolution: :hour
+  #   Aeternitas::Metrics.execution_times MyPollable
+  # @see #get
+  module Metrics
+    AVAILABLE_METRICS = {
+      polls: :counter,
+      successful_polls: :counter,
+      failed_polls: :counter,
+      ignored_errors: :counter,
+      deactivations: :counter,
+      execution_time: :value,
+      guard_locked: :counter,
+      guard_timeout: :value,
+      guard_timeout_exceeded: :counter,
+      sources_created: :counter,
+      pollables_created: :counter
+    }.freeze
+
+    TabsTabs.configure do |tabstabs_config|
+      tabstabs_config.unregister_resolutions(:week, :month, :year)
+
+      tabstabs_config.register_resolution Aeternitas::Metrics::TenMinutesResolution
+
+      tabstabs_config.set_expirations(
+        minute: 3.days,
+        ten_minutes: 14.days,
+        hour: 2.months
+      )
+    end
+
+    AVAILABLE_METRICS.each_pair do |metric, _|
+      module_eval <<-eoruby, __FILE__, __LINE__ + 1
+        def self.#{metric}(pollable, from: 1.hour.ago, to: Time.now, resolution: :minute)
+          self.get(:#{metric}, pollable, from: from, to: to, resolution: resolution )
+        end
+      eoruby
+    end
+
+    # Increses the specified counter metric for the given pollable.
+    # @param [Symbol, String] name the metric
+    # @param [Pollable] pollable_class pollable instance
+    def self.log(name, pollable_class)
+      raise('Metric not found') unless AVAILABLE_METRICS.key? name
+      raise ArgumentError, "#{name} isn't a Counter" unless AVAILABLE_METRICS[name] == :counter
+      begin
+        TabsTabs.increment_counter(get_key(name, pollable_class))
+        TabsTabs.increment_counter(get_key(name, Aeternitas::Pollable))
+      rescue StandardError ; end
+    end
+
+    # Logs a value in a value metric for the given pollable.
+    # @param [Symbol String] name the metric
+    # @param [Pollable] pollable_class pollable instance
+    # @param [Object] value the value
+    def self.log_value(name, pollable_class, value)
+      raise('Metric not found') unless AVAILABLE_METRICS.key? name
+      raise(ArgumentError, "#{name} isn't a Value") unless AVAILABLE_METRICS[name] == :value
+      begin
+        TabsTabs.record_value(get_key(name, pollable_class), value)
+        TabsTabs.record_value(get_key(name, Aeternitas::Pollable), value)
+      rescue StandardError ; end
+    end
+
+    # Retrieves the stats of the given metric in the given time frame and resolution.
+    # @param [Symbol String] name the metric
+    # @param [Object] pollable_class the pollable class
+    # @param [DateTime] from begin of the time frame
+    # @param [DateTime] to end of the timeframe
+    # @param [Symbol] resolution resolution
+    # @return [Aeternitas::Metrics::Counter, Aeternitas::Metrics::Value] stats
+    def self.get(name, pollable_class, from: 1.hour.ago, to: Time.now, resolution: :minute)
+      raise('Metric not found') unless AVAILABLE_METRICS.key? name
+      raise('Invalid interval') if from > to
+      result = TabsTabs.get_stats(get_key(name, pollable_class), from..to, resolution)
+      if AVAILABLE_METRICS[name] == :counter
+        Counter.new(result)
+      else
+        Values.new(result)
+      end
+    rescue TabsTabs::UnknownMetricError => _
+      TabsTabs.create_metric(get_key(name, pollable_class), AVAILABLE_METRICS[name].to_s)
+      get(name, pollable_class, from: from, to: to, resolution: resolution)
+    end
+
+    # Returns the failure ratio of the given job for given time frame and resolution
+    # @param [Symbol String] name the metric
+    # @param [Object] pollable_class the pollable class
+    # @param [DateTime] from begin of the time frame
+    # @param [DateTime] to end of the timeframe
+    # @param [Symbol] resolution resolution
+    # @return [Aeternitas::Metrics::Ratio] ratio time series
+    def self.failure_ratio(pollable_class, from: 1.hour.ago, to: Time.now, resolution: :minute)
+      polls = polls(pollable_class, from: from, to: to, resolution: resolution)
+      failed_polls = failed_polls(pollable_class, from: from, to: to, resolution: resolution)
+      Ratio.new(from, to, resolution, calculate_ratio(polls, failed_polls))
+    end
+
+    # Returns the lock ratio of the given job for given time frame and resolution
+    # @param [Symbol String] name the metric
+    # @param [Object] pollable_class the pollable class
+    # @param [DateTime] from begin of the time frame
+    # @param [DateTime] to end of the timeframe
+    # @param [Symbol] resolution resolution
+    # @return [Aeternitas::Metrics::Ratio] ratio time series
+    def self.guard_locked_ratio(pollable_class, from: 1.hour.ago, to: Time.now, resolution: :minute)
+      polls = polls(pollable_class, from: from, to: to, resolution: resolution)
+      guard_locked = guard_locked(pollable_class, from: from, to: to, resolution: resolution)
+      Ratio.new(from, to, resolution, calculate_ratio(polls, guard_locked))
+    end
+
+    # Computes the metric key of a given metric-pollable pair
+    # @param [Symbol, String] name the metric
+    # @param [Object] pollable_class pollable class
+    # @return [String] the metric key
+    def self.get_key(name, pollable_class)
+      "#{name}:#{pollable_class.name}"
+    end
+
+    # Computes the ratio of a base counter time series and a target counter time series
+    # @param [Array] base base time series data
+    # @param [Array] target target time series data
+    # @return [Array] ratio time series data
+    def self.calculate_ratio(base, target)
+      base.zip(target).map do |b, t|
+        {
+          timestamp: b['timestamp'],
+          ratio: b['count'].to_i.zero? ? 0 : t['count'].to_i / b['count'].to_f
+        }.with_indifferent_access
+      end
+    end
+  end
+end