apisonator 2.100.0

data/lib/3scale/backend/analytics/redshift/importer.rb

@@ -0,0 +1,83 @@
+module ThreeScale
+  module Backend
+    module Analytics
+      module Redshift
+
+        # The main responsibility of this class is to schedule jobs that import
+        # events that are stored in S3 into Redshift.
+        # We know that the distributed locking algorithm that we are using
+        # guarantees that two jobs will not be running at the same time except
+        # in some corner cases, like in the case of a failure of one of the Redis
+        # masters. However, this is not a problem in our case. If two Redshift
+        # jobs run at the same time, they will try to import the same S3 paths
+        # from Redshift. This is not a problem because the import method that
+        # we use ensures that we do not import duplicates into Redshift.
+        # Check the Redshift::Adapter class for more details on this.
+        class Importer
+          TTL_JOB_RUNNING_KEY_SEC = 60*60
+          private_constant :TTL_JOB_RUNNING_KEY_SEC
+
+          REDSHIFT_ENABLED_KEY = 'redshift:enabled'.freeze
+          private_constant :REDSHIFT_ENABLED_KEY
+
+          class << self
+            def schedule_job
+              if enabled? && Backend.production?
+                lock_key = dist_lock.lock
+                if lock_key
+                  Resque.enqueue(Job, lock_key, Time.now.utc.to_f)
+                end
+              end
+            end
+
+            # Returns a UTC time that represents the hour when the newest events
+            # imported in Redshift were generated or nil if nothing has been
+            # imported.
+            def latest_imported_events_time
+              latest_timestamp = db_adapter.latest_timestamp_read
+              return nil if latest_timestamp.nil?
+              DateTime.parse(latest_timestamp).to_time.utc
+            end
+
+            def consistent_data?
+              db_adapter.consistent_data?
+            end
+
+            def enable
+              storage.set(REDSHIFT_ENABLED_KEY, '1')
+            end
+
+            def disable
+              storage.del(REDSHIFT_ENABLED_KEY)
+            end
+
+            def enabled?
+              storage.get(REDSHIFT_ENABLED_KEY).to_i == 1
+            end
+
+            # To be called by from a Redshift job once it exits so other jobs can run
+            def job_finished(lock_key)
+              dist_lock.unlock if lock_key == dist_lock.current_lock_key
+            end
+
+            private
+
+            def storage
+              Backend::Storage.instance
+            end
+
+            def dist_lock
+              @dist_lock ||= DistributedLock.new(self.name,
+                                                 TTL_JOB_RUNNING_KEY_SEC,
+                                                 storage)
+            end
+
+            def db_adapter
+              Adapter
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/analytics/redshift/job.rb

@@ -0,0 +1,33 @@
+module ThreeScale
+  module Backend
+    module Analytics
+      module Redshift
+        class Job < BackgroundJob
+          @queue = :stats
+
+          class << self
+            def perform_logged(lock_key, _enqueue_time)
+              begin
+                latest_time_inserted = Adapter.insert_pending_events
+                ok = true
+                msg = job_ok_msg(latest_time_inserted)
+              rescue Exception => e
+                ok = false
+                msg = e.message
+              end
+
+              Importer.job_finished(lock_key)
+              [ok, msg]
+            end
+
+            private
+
+            def job_ok_msg(time_utc)
+              "Events imported correctly. Latest ones are from: #{time_utc.to_s}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/application.rb

@@ -0,0 +1,330 @@
+module ThreeScale
+  module Backend
+    class Application
+      include Storable
+
+      # list of attributes to be fetched from storage
+      ATTRIBUTES = [:state, :plan_id, :plan_name, :redirect_url,
+                    :user_required].freeze
+      private_constant :ATTRIBUTES
+
+      attr_accessor :service_id, :id, *ATTRIBUTES
+      attr_writer :metric_names
+
+      def to_hash
+        {
+          service_id: service_id,
+          id: id,
+          state: state,
+          plan_id: plan_id,
+          plan_name: plan_name,
+          redirect_url: redirect_url,
+          user_required: user_required
+        }
+      end
+
+      def update(attributes)
+        attributes.each do |attr, val|
+          public_send("#{attr}=", val)
+        end
+        self
+      end
+
+      class << self
+        include Memoizer::Decorator
+
+        def attribute_names
+          (ATTRIBUTES + %i[service_id id metric_names].freeze).freeze
+        end
+
+        def load(service_id, id)
+          return nil unless service_id and id
+          values = storage.mget(storage_key(service_id, id, :state),
+                                storage_key(service_id, id, :plan_id),
+                                storage_key(service_id, id, :plan_name),
+                                storage_key(service_id, id, :redirect_url),
+                                storage_key(service_id, id, :user_required))
+          state, plan_id, plan_name, redirect_url, user_required = values
+
+          # save a network call by just checking state here for existence
+          return nil unless state
+
+          ## the default value is false
+          user_required = user_required.to_i > 0
+
+          new(service_id: service_id,
+              id: id,
+              state: state.to_sym,
+              plan_id: plan_id,
+              plan_name: plan_name,
+              user_required: user_required,
+              redirect_url: redirect_url)
+        end
+        memoize :load
+
+        def load!(service_id, app_id)
+          load(service_id, app_id) or raise ApplicationNotFound, app_id
+        end
+        memoize :load!
+
+        def load_id_by_key(service_id, key)
+          storage.get(id_by_key_storage_key(service_id, key))
+        end
+        memoize :load_id_by_key
+
+        def save_id_by_key(service_id, key, id)
+          raise ApplicationHasInconsistentData.new(id, key) if [service_id, id, key].any?(&:blank?)
+          storage.set(id_by_key_storage_key(service_id, key), id).tap do
+            Memoizer.memoize(Memoizer.build_key(self, :load_id_by_key, service_id, key), id)
+          end
+        end
+
+        def delete_id_by_key(service_id, key)
+          storage.del(id_by_key_storage_key(service_id, key)).tap do
+            Memoizer.clear(Memoizer.build_key(self, :load_id_by_key, service_id, key))
+          end
+        end
+
+        def load_by_id_or_user_key!(service_id, app_id, user_key)
+          with_app_id_from_params service_id, app_id, user_key do |appid|
+            load service_id, appid
+          end
+        end
+
+        def extract_id!(service_id, app_id, user_key, access_token)
+          with_app_id_from_params service_id, app_id, user_key, access_token do |appid|
+            exists? service_id, appid and appid
+          end
+        end
+
+        def exists?(service_id, id)
+          storage.exists(storage_key(service_id, id, :state))
+        end
+        memoize :exists?
+
+        def delete(service_id, id)
+          raise ApplicationNotFound, id unless exists?(service_id, id)
+          delete_data service_id, id
+          clear_cache service_id, id
+          OAuth::Token::Storage.remove_tokens(service_id, id)
+        end
+
+        def delete_data(service_id, id)
+          storage.pipelined do
+            delete_set(service_id, id)
+            delete_attributes(service_id, id)
+          end
+        end
+
+        def clear_cache(service_id, id)
+          params = [service_id, id]
+          keys = Memoizer.build_keys_for_class(self,
+                    load: params,
+                    load!: params,
+                    exists?: params)
+          Memoizer.clear keys
+        end
+
+        def applications_set_key(service_id)
+          encode_key("service_id:#{service_id}/applications")
+        end
+
+        def save(attributes)
+          application = new(attributes)
+          application.save
+          application
+        end
+
+        def storage_key(service_id, id, attribute)
+          encode_key("application/service_id:#{service_id}/id:#{id}/#{attribute}")
+        end
+
+        private
+
+        def id_by_key_storage_key(service_id, key)
+          encode_key("application/service_id:#{service_id}/key:#{key}/id")
+        end
+
+        def delete_set(service_id, id)
+          storage.srem(applications_set_key(service_id), id)
+        end
+
+        def delete_attributes(service_id, id)
+          storage.del(
+            ATTRIBUTES.map do |f|
+              storage_key(service_id, id, f)
+            end
+          )
+        end
+
+        def with_app_id_from_params(service_id, app_id, user_key, access_token = nil)
+          if app_id
+            raise AuthenticationError unless user_key.nil?
+          elsif user_key
+            app_id = load_id_by_key(service_id, user_key)
+            raise UserKeyInvalid, user_key if app_id.nil?
+          elsif access_token
+            app_id, * = OAuth::Token::Storage.get_credentials access_token, service_id
+          else
+            raise ApplicationNotFound
+          end
+
+          yield app_id or raise ApplicationNotFound, app_id
+        end
+      end
+
+      def user_required?
+        @user_required
+      end
+
+      def save
+        raise ApplicationHasNoState.new(id) if !state
+
+        storage.pipelined do
+          persist_attributes
+          persist_set
+        end
+
+        self.class.clear_cache(service_id, id)
+
+        Memoizer.memoize(Memoizer.build_key(self.class, :exists?, service_id, id), state)
+      end
+
+      def invalidate_cache(cmds, cache_key)
+        # cache key cannot be just cache_key.
+        # Command must be added to avoid collisions between different ops over same key
+        cmds.each { |cmd| Memoizer.clear(Memoizer.build_key(self.class, cmd, cache_key)) }
+      end
+
+      def storage_key(attribute)
+        self.class.storage_key(service_id, id, attribute)
+      end
+
+      def applications_set_key(service_id)
+        self.class.applications_set_key(service_id)
+      end
+
+      def metric_names
+        @metric_names ||= {}
+      end
+
+      def metric_name(metric_id)
+        metric_names[metric_id] ||= Metric.load_name(service_id, metric_id)
+      end
+
+      # Sets @metric_names with the names of all the metrics for which there is
+      # a usage limit that applies to the app, and returns it.
+      def load_metric_names
+        metric_ids = usage_limits.map(&:metric_id)
+        @metric_names = Metric.load_all_names(service_id, metric_ids)
+      end
+
+      def usage_limits
+        @usage_limits ||= UsageLimit.load_all(service_id, plan_id)
+      end
+
+      def active?
+        state == :active
+      end
+
+      #
+      # KEYS
+      #
+
+      def keys
+        # We memoize with self.class to avoid caching the result for specific
+        # instances as opposed to the combination of service_id and app_id.
+        db_key = storage_key(:keys)
+        key = Memoizer.build_key(self.class, :smembers, db_key)
+        Memoizer.memoize_block(key) do
+          storage.smembers(db_key)
+        end
+      end
+
+      # Create new application key and add it to the list of keys of this app.
+      # If value is nil, generates new random key, otherwise uses the given
+      # value as the new key.
+      def create_key(value = nil)
+        db_key = storage_key(:keys)
+        invalidate_cache([:smembers, :scard], db_key)
+        value ||= SecureRandom.hex(16)
+        storage.sadd(db_key, value)
+        value
+      end
+
+      def delete_key(value)
+        db_key = storage_key(:keys)
+        invalidate_cache([:smembers, :scard, :sismember], db_key)
+        storage.srem(db_key, value)
+      end
+
+      def has_keys?
+        db_key = storage_key(:keys)
+        key = Memoizer.build_key(self.class, :scard, db_key)
+        Memoizer.memoize_block(key) do
+          storage.scard(db_key).to_i > 0
+        end
+      end
+
+      def has_no_keys?
+        !has_keys?
+      end
+
+      def has_key?(value)
+        db_key = storage_key(:keys)
+        key = Memoizer.build_key(self.class, :sismember, db_key, value)
+        Memoizer.memoize_block(key) do
+          storage.sismember(db_key, value)
+        end
+      end
+
+      #
+      # REFERRER FILTER
+      #
+
+      def referrer_filters
+        db_key = storage_key(:referrer_filters)
+        key = Memoizer.build_key(self.class, :smembers, db_key)
+        Memoizer.memoize_block(key) do
+          storage.smembers(db_key)
+        end
+      end
+
+      def create_referrer_filter(value)
+        raise ReferrerFilterInvalid, "referrer filter can't be blank" if value.blank?
+        db_key = storage_key(:referrer_filters)
+        invalidate_cache([:smembers, :scard], db_key)
+        storage.sadd(db_key, value)
+        value
+      end
+
+      def delete_referrer_filter(value)
+        db_key = storage_key(:referrer_filters)
+        invalidate_cache([:smembers, :scard], db_key)
+        storage.srem(db_key, value)
+      end
+
+      def has_referrer_filters?
+        db_key = storage_key(:referrer_filters)
+        key = Memoizer.build_key(self.class, :scard, db_key)
+        Memoizer.memoize_block(key) do
+          storage.scard(db_key).to_i > 0
+        end
+      end
+
+      private
+
+      def persist_attributes
+        storage.set(storage_key(:state), state.to_s) if state
+        storage.set(storage_key(:plan_id), plan_id) if plan_id
+        storage.set(storage_key(:plan_name), plan_name) if plan_name
+        storage.set(storage_key(:user_required), user_required? ? 1 : 0)
+        storage.set(storage_key(:redirect_url), redirect_url) if redirect_url
+      end
+
+      def persist_set
+        storage.sadd(applications_set_key(service_id), id)
+      end
+    end
+  end
+end

data/lib/3scale/backend/application_events.rb

@@ -0,0 +1,76 @@
+require '3scale/backend/storage'
+require '3scale/backend/event_storage'
+require '3scale/backend/stats/keys'
+
+module ThreeScale
+  module Backend
+    class ApplicationEvents
+
+      class << self
+        include Backend::StorageKeyHelpers
+      end
+
+      DAILY_KEY_TTL = 172_800
+
+      Error = Class.new StandardError
+      class PingFailed < Error
+        def initialize(e)
+          super "Application event ping failed: #{e.class} - #{e.message}"
+        end
+      end
+
+      # ping the frontend if any event is pending for processing
+      def self.ping
+        EventStorage.ping_if_not_empty
+      rescue => e
+        raise PingFailed.new(e)
+      end
+
+      def self.generate(applications)
+        return if applications.nil? || applications.empty?
+        applications.each do |application|
+          service_id     = application[:service_id]
+          application_id = application[:application_id]
+
+          first_traffic(service_id, application_id)
+          first_daily_traffic(service_id, application_id)
+        end
+      end
+
+      private
+
+      def self.first_traffic(service_id, application_id)
+        key = Stats::Keys.applications_key_prefix(
+          Stats::Keys.service_key_prefix(service_id)
+        )
+        if storage.sadd(key, encode_key(application_id))
+          EventStorage.store(:first_traffic,
+                             { service_id:     service_id,
+                               application_id: application_id,
+                               timestamp:      Time.now.utc.to_s })
+        end
+      end
+
+      def self.first_daily_traffic(service_id, application_id)
+        timestamp = Time.now.utc
+        day_key   = Period::Boundary.day_start(timestamp).to_compact_s
+        daily_key = "daily_traffic/service:#{service_id}/" \
+                    "cinstance:#{application_id}/#{day_key}"
+
+        Memoizer.memoize_block(daily_key) do
+          if storage.incr(daily_key) == 1
+            storage.expire(daily_key, DAILY_KEY_TTL)
+            EventStorage.store(:first_daily_traffic,
+                               { service_id:     service_id,
+                                 application_id: application_id,
+                                 timestamp:      timestamp.to_s })
+          end
+        end
+      end
+
+      def self.storage
+        Storage.instance
+      end
+    end
+  end
+end