apisonator 2.100.0

data/lib/3scale/backend/service_token.rb

@@ -0,0 +1,97 @@
+module ThreeScale
+  module Backend
+    class ServiceToken
+
+      module KeyHelpers
+        def key(service_token, service_id)
+          encode_key("service_token/token:#{service_token}/service_id:#{service_id}")
+        end
+      end
+
+      include KeyHelpers
+      extend KeyHelpers
+      include Storable
+
+      ValidationError = Class.new(ThreeScale::Backend::Invalid)
+
+      class InvalidServiceToken < ValidationError
+        def initialize
+          super('Service token cannot be blank'.freeze)
+        end
+      end
+
+      class InvalidServiceId < ValidationError
+        def initialize
+          super('Service ID cannot be blank'.freeze)
+        end
+      end
+
+      # We want to use a hash in Redis because in the future we might have
+      # several fields related to permissions and roles.
+      # For now we do not need any of those fields, but we need to define at
+      # least one to be able to create a hash, even if we are not going to use
+      # it.
+      PERMISSIONS_KEY_FIELD = 'permissions'.freeze
+      private_constant :PERMISSIONS_KEY_FIELD
+
+      class << self
+        include Memoizer::Decorator
+
+        def save(service_token, service_id)
+          validate_pairs([{ service_token: service_token, service_id: service_id }])
+          storage.hset(key(service_token, service_id), PERMISSIONS_KEY_FIELD, ''.freeze)
+        end
+
+        # Saves a collection of (service_token, service_id) pairs only if all
+        # the pairs contain valid data, meaning that there are no null or empty
+        # strings.
+        def save_pairs(token_id_pairs)
+          validate_pairs(token_id_pairs)
+
+          token_id_pairs.each do |pair|
+            unchecked_save(pair[:service_token], pair[:service_id])
+          end
+        end
+
+        def delete(service_token, service_id)
+          res = storage.del(key(service_token, service_id))
+          clear_cache(service_token, service_id)
+          res
+        end
+
+        def exists?(service_token, service_id)
+          storage.exists(key(service_token, service_id))
+        end
+        memoize :exists?
+
+        private
+
+        def validate_pairs(token_id_pairs)
+          invalid_token = token_id_pairs.any? do |pair|
+            pair[:service_token].nil? || pair[:service_token].empty?
+          end
+          raise InvalidServiceToken if invalid_token
+
+          invalid_service_id = token_id_pairs.any? do |pair|
+            pair[:service_id].nil? || pair[:service_id].to_s.empty?
+          end
+          raise InvalidServiceId if invalid_service_id
+        end
+
+        def unchecked_save(service_token, service_id)
+          storage.hset(key(service_token, service_id), PERMISSIONS_KEY_FIELD, ''.freeze)
+        end
+
+        def storage
+          Storage.instance
+        end
+
+        def clear_cache(service_token, service_id)
+          Memoizer.clear(Memoizer.build_key(self, :exists?, service_token, service_id))
+        end
+
+      end
+
+    end
+  end
+end

data/lib/3scale/backend/stats.rb

@@ -0,0 +1,8 @@
+require '3scale/backend/stats/codes_commons'
+require '3scale/backend/stats/period_commons'
+require '3scale/backend/stats/aggregator'
+require '3scale/backend/stats/delete_job_def'
+require '3scale/backend/stats/key_generator'
+require '3scale/backend/stats/partition_generator_job'
+require '3scale/backend/stats/partition_eraser_job'
+require '3scale/backend/stats/cleaner'

data/lib/3scale/backend/stats/aggregator.rb

@@ -0,0 +1,170 @@
+require '3scale/backend/logging'
+require '3scale/backend/stats/storage'
+require '3scale/backend/stats/keys'
+require '3scale/backend/application_events'
+require '3scale/backend/transaction'
+require '3scale/backend/stats/aggregators/response_code'
+require '3scale/backend/stats/aggregators/usage'
+
+module ThreeScale
+  module Backend
+    module Stats
+
+      # This class contains several methods that deal with buckets, which are
+      # only used in the SaaS analytics system.
+      class Aggregator
+        # We need to limit the number of buckets stored in the system.
+        # The reason is that our Redis can grow VERY quickly if we start
+        # creating buckets and we never delete them.
+        # When the max defined is reached, I simply disable the option
+        # to save the stats keys in buckets. Yes, we will lose data,
+        # but that is better than the alternative. We will try to find
+        # a better alternative once we cannot afford to lose data.
+        # Right now, we are just deleting the stats keys with
+        # period = minute, so we can restore everything else.
+        MAX_BUCKETS = 360
+        private_constant :MAX_BUCKETS
+
+        MAX_BUCKETS_CREATED_MSG =
+            'Bucket creation has been disabled. Max number of stats buckets reached'.freeze
+        private_constant :MAX_BUCKETS_CREATED_MSG
+
+        class << self
+          include Backend::StorageKeyHelpers
+          include Configurable
+          include Keys
+          include Logging
+
+          # This method stores the events in buckets if that option is enabled
+          # or if it was disable because of an emergency (not because a user
+          # did it manually), and Kinesis has already consumed all the pending
+          # buckets.
+          def process(transactions)
+            current_bucket = nil
+
+            if configuration.can_create_event_buckets
+              # Only disable indicating emergency if bucket storage is enabled.
+              # Otherwise, we might indicate emergency when a user manually
+              # disabled it previously.
+              if Storage.enabled? && buckets_limit_exceeded?
+                Storage.disable!(true)
+                log_bucket_creation_disabled
+              elsif save_in_bucket?
+                Storage.enable! unless Storage.enabled?
+                current_bucket = Time.now.utc.beginning_of_bucket(stats_bucket_size)
+                                             .to_not_compact_s
+              end
+            end
+
+            touched_apps = aggregate(transactions, current_bucket)
+
+            ApplicationEvents.generate(touched_apps.values)
+            update_alerts(touched_apps)
+            begin
+              ApplicationEvents.ping
+            rescue ApplicationEvents::PingFailed => e
+              # we could not ping the frontend, log it
+              logger.notify e
+            end
+          end
+
+          private
+
+          # Aggregate stats values for a collection of Transactions.
+          #
+          # @param [Array] transactions the collection of transactions
+          # @param [String, Nil] bucket
+          # @return [Hash] A Hash where each key is an application_id and the
+          #   value is another Hash with service_id and application_id.
+          def aggregate(transactions, bucket = nil)
+            touched_apps = {}
+
+            transactions.each_slice(PIPELINED_SLICE_SIZE) do |slice|
+              storage.pipelined do
+                slice.each do |transaction|
+                  aggregate_all(transaction, bucket)
+                  touched_apps.merge!(touched_relation(transaction))
+                end
+              end
+            end
+
+            touched_apps
+          end
+
+          def aggregate_all(transaction, bucket)
+            [Aggregators::ResponseCode, Aggregators::Usage].each do |aggregator|
+              aggregator.aggregate(transaction, bucket)
+            end
+          end
+
+          def save_in_bucket?
+            if Storage.enabled?
+              true
+            else
+              Storage.last_disable_was_emergency? && bucket_storage.pending_buckets_size == 0
+            end
+          end
+
+          def stats_bucket_size
+            @stats_bucket_size ||= (configuration.stats.bucket_size || 5)
+          end
+
+          def storage
+            Backend::Storage.instance
+          end
+
+          def bucket_storage
+            Stats::Storage.bucket_storage
+          end
+
+          # Return a Hash with needed info to update usages and alerts.
+          #
+          # @param [Transaction] transaction
+          # @return [Hash] the hash that contains the application_id that has
+          #   been updated and the transaction's service_id. The key of the
+          #   hash is the application_id.
+          def touched_relation(transaction)
+            relation_value = transaction.send(:application_id)
+            { relation_value => { application_id: relation_value,
+                                  service_id: transaction.service_id } }
+          end
+
+          def buckets_limit_exceeded?
+            bucket_storage.pending_buckets_size > MAX_BUCKETS
+          end
+
+          def log_bucket_creation_disabled
+            logger.info(MAX_BUCKETS_CREATED_MSG)
+          end
+
+          def update_alerts(applications)
+            current_timestamp = Time.now.getutc
+
+            applications.each do |_appid, values|
+              service_id = values[:service_id]
+              application = Backend::Application.load(service_id,
+                                                      values[:application_id])
+
+              application.load_metric_names
+              usage = Usage.application_usage(application, current_timestamp)
+              status = Transactor::Status.new(service_id: service_id,
+                                              application: application,
+                                              values: usage)
+
+              max_utilization, max_record = Alerts.utilization(
+                  status.application_usage_reports)
+
+              if max_utilization >= 0.0
+                Alerts.update_utilization(service_id,
+                                          values[:application_id],
+                                          max_utilization,
+                                          max_record,
+                                          current_timestamp)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/aggregators/base.rb

@@ -0,0 +1,72 @@
+module ThreeScale
+  module Backend
+    module Stats
+      module Aggregators
+        module Base
+          # Aggregates a value in a timestamp for all given keys using a specific
+          # Redis command to store them. If a bucket_key is specified, each key will
+          # be added to a Redis Set with that name.
+          #
+          # @param [Integer] value
+          # @param [Time] timestamp
+          # @param [Array] keys array of {(service|application|user) => "key"}
+          # @param [Symbol] cmd
+          # @param [String, Nil] bucket
+          def aggregate_values(value, timestamp, keys, cmd, bucket)
+            keys_for_bucket = []
+
+            keys.each do |metric_type, prefix_key|
+              granularities(metric_type).each do |granularity|
+                key = counter_key(prefix_key, granularity.new(timestamp))
+                expire_time = Stats::PeriodCommons.expire_time_for_granularity(granularity)
+
+                store_key(cmd, key, value, expire_time)
+
+                unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity)
+                  keys_for_bucket << key
+                end
+              end
+            end
+
+            store_in_changed_keys(keys_for_bucket, bucket) if bucket
+          end
+
+          # Return Redis command depending on raw_value.
+          # If raw_value is a string with a '#' in the beginning, it returns 'set'.
+          # Else, it returns 'incrby'.
+          #
+          # @param [String] raw_value
+          # @return [Symbol] the Redis command
+          def storage_cmd(raw_value)
+            Backend::Usage.is_set?(raw_value) ? :set : :incrby
+          end
+
+          def storage
+            Backend::Storage.instance
+          end
+
+          protected
+
+          def granularities(metric_type)
+            metric_type == :service ? Stats::PeriodCommons::SERVICE_GRANULARITIES : Stats::PeriodCommons::EXPANDED_GRANULARITIES
+          end
+
+          def store_key(cmd, key, value, expire_time = nil)
+            storage.send(cmd, key, value)
+            storage.expire(key, expire_time) if expire_time
+          end
+
+          def store_in_changed_keys(keys, bucket)
+            bucket_storage.put_in_bucket(keys, bucket)
+          end
+
+          private
+
+          def bucket_storage
+            Stats::Storage.bucket_storage
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/aggregators/response_code.rb

@@ -0,0 +1,58 @@
+require '3scale/backend/stats/keys'
+require '3scale/backend/transaction'
+require '3scale/backend/stats/aggregators/base'
+
+module ThreeScale
+  module Backend
+    module Stats
+      module Aggregators
+        class ResponseCode
+          class << self
+            include Keys
+            include Base
+
+            def aggregate(transaction, bucket = nil)
+              keys_for_multiple_codes = keys_for_response_code(transaction)
+              timestamp = transaction.timestamp
+
+              # For now, we are not interested in storing response codes in
+              # buckets, that is the reason why set bucket to nil.
+              bucket = nil
+
+              keys_for_multiple_codes.each do |keys|
+                aggregate_values(1, timestamp, keys, :incrby, bucket)
+              end
+            end
+
+
+            protected
+            def keys_for_response_code(transaction)
+              response_code = transaction.extract_response_code
+              return {} unless response_code
+              values = values_to_inc(response_code)
+              values.flat_map do |code|
+                Keys.transaction_keys(transaction, :response_code, code)
+              end
+            end
+
+            def values_to_inc(response_code)
+              group_code = Stats::CodesCommons.get_http_code_group(response_code)
+              [].tap do |keys|
+                keys << group_code if tracked_group_code?(group_code)
+                keys << response_code.to_s if tracked_code?(response_code)
+              end
+            end
+
+            def tracked_code?(code)
+              Stats::CodesCommons::TRACKED_CODES.include?(code)
+            end
+
+            def tracked_group_code?(group_code)
+              Stats::CodesCommons::TRACKED_CODE_GROUPS.include?(group_code)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/aggregators/usage.rb

@@ -0,0 +1,34 @@
+require '3scale/backend/stats/keys'
+require '3scale/backend/transaction'
+require '3scale/backend/stats/aggregators/base'
+
+module ThreeScale
+  module Backend
+    module Stats
+      module Aggregators
+        class Usage
+          class << self
+            include Keys
+            include Base
+
+            # Aggregates the usage of a transaction. If a bucket time is specified,
+            # all new or updated stats keys will be stored in a Redis Set.
+            #
+            # @param [Transaction] transaction
+            # @param [String, Nil] bucket
+            def aggregate(transaction, bucket = nil)
+              transaction.usage.each do |metric_id, raw_value|
+                metric_keys = Keys.transaction_keys(transaction, :metric, metric_id)
+                cmd         = storage_cmd(raw_value)
+                value       = Backend::Usage.get_from raw_value
+
+                aggregate_values(value, transaction.timestamp, metric_keys, cmd, bucket)
+              end
+            end
+
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/bucket_reader.rb

@@ -0,0 +1,135 @@
+module ThreeScale
+  module Backend
+    module Stats
+
+      # This class allows us to read the buckets that we are creating in Redis
+      # to store the stats keys that change. It also allows us to keep track of
+      # the ones that are pending to be read.
+      class BucketReader
+
+        # This private nested class allows us to isolate accesses to Redis.
+        class LatestBucketReadMarker
+          LATEST_BUCKET_READ_KEY = 'send_to_kinesis:latest_bucket_read'
+          private_constant :LATEST_BUCKET_READ_KEY
+
+          def initialize(storage)
+            @storage = storage
+          end
+
+          def latest_bucket_read=(latest_bucket_read)
+            storage.set(LATEST_BUCKET_READ_KEY, latest_bucket_read)
+          end
+
+          def latest_bucket_read
+            storage.get(LATEST_BUCKET_READ_KEY)
+          end
+
+          private
+
+          attr_reader :storage
+        end
+        private_constant :LatestBucketReadMarker
+
+        # Before we read and mark a bucket as read, we need to make sure that
+        # it will not receive more events. Otherwise, there is the risk that
+        # we will miss some events.
+        # Buckets are created every 'bucket_create_interval' seconds, it is one
+        # of the parameters that 'initialize' receives. We should be able to
+        # read any bucket identified with a timestamp ts, where
+        # ts < Time.now - bucket_create_interval. However, in order to be sure
+        # that we will not miss any events, we are going to define a constant
+        # that will define some backup time.
+        BACKUP_SECONDS_READ_BUCKET = 10
+        private_constant :BACKUP_SECONDS_READ_BUCKET
+
+        KEYS_SLICE_CALL_TO_REDIS = 1000
+        private_constant :KEYS_SLICE_CALL_TO_REDIS
+
+        InvalidInterval = Class.new(ThreeScale::Backend::Error)
+
+        def initialize(bucket_create_interval, bucket_storage, events_storage)
+          # This is needed because ThreeScale::Backend::TimeHacks.beginning_of_bucket
+          if 60%bucket_create_interval != 0 || bucket_create_interval <= 0
+            raise InvalidInterval, 'Bucket create interval needs to divide 60'
+          end
+
+          @bucket_create_interval = bucket_create_interval
+          @bucket_storage = bucket_storage
+          @events_storage = events_storage
+          @latest_bucket_read_marker = LatestBucketReadMarker.new(bucket_storage.storage)
+        end
+
+        # Returns the pending events and the bucket of the most recent of the
+        # events sent. This allows the caller to call latest_bucket_read= when
+        # it has processed all the events.
+        def pending_events_in_buckets(end_time_utc: Time.now.utc, max_buckets: nil)
+          buckets = if max_buckets
+                      pending_buckets(end_time_utc).take(max_buckets)
+                    else
+                      pending_buckets(end_time_utc).to_a
+                    end
+
+          { events: events(buckets), latest_bucket: buckets.last }
+        end
+
+        def latest_bucket_read=(latest_bucket_read)
+          latest_bucket_read_marker.latest_bucket_read = latest_bucket_read
+        end
+
+        private
+
+        attr_reader :bucket_create_interval,
+                    :bucket_storage,
+                    :events_storage,
+                    :latest_bucket_read_marker
+
+        def pending_buckets(end_time_utc = Time.now.utc)
+          latest_bucket_read = latest_bucket_read_marker.latest_bucket_read
+          start_time = unless latest_bucket_read.nil?
+                         bucket_to_time(latest_bucket_read) + bucket_create_interval
+                       end
+          end_time = end_time_with_backup(end_time_utc)
+          stored_buckets(start_time, end_time)
+        end
+
+        def events(buckets)
+          event_keys = bucket_storage.content(buckets)
+
+          # Values are stored as strings in Redis, but we want integers.
+          # There are some values that can be nil. This happens when the key
+          # has a TTL and we read it once it has expired. Right now, event keys
+          # with granularity = 'minute' expire after 180 s (see
+          # Stats::Aggregators::Base module). We might need to increase that to
+          # make sure that we do not miss any values.
+          event_values = event_keys.each_slice(KEYS_SLICE_CALL_TO_REDIS)
+                                   .flat_map do |keys_slice|
+            events_storage.mget(keys_slice)
+          end.map { |value| Integer(value) if value }
+
+          Hash[event_keys.zip(event_values)]
+        end
+
+        def end_time_with_backup(end_time_utc)
+          [end_time_utc, Time.now.utc - bucket_create_interval - BACKUP_SECONDS_READ_BUCKET].min
+        end
+
+        def stored_buckets(start_time_utc, end_time_utc)
+          range = { }
+          range[:first] = time_to_bucket_name(start_time_utc) if start_time_utc
+          range[:last] = time_to_bucket_name(end_time_utc)
+          bucket_storage.buckets(range)
+        end
+
+        def time_to_bucket_name(time_utc)
+          # We know that the names of the buckets follow the following pattern:
+          # they are a timestamp with format YYYYmmddHHMMSS
+          time_utc.strftime('%Y%m%d%H%M%S')
+        end
+
+        def bucket_to_time(bucket_name)
+          DateTime.parse(bucket_name).to_time.utc
+        end
+      end
+    end
+  end
+end