apisonator 2.100.0

data/lib/3scale/backend/stats/bucket_storage.rb

@@ -0,0 +1,108 @@
+module ThreeScale
+  module Backend
+    module Stats
+
+      # This class manages the buckets where we are storing stats keys.
+      # The way those buckets work is as follows: we are creating a bucket
+      # every few seconds (10 by default now), and in each of those buckets,
+      # we store all the stats keys that have changed in that bucket creation
+      # interval.
+      # The values of the keys that are stored in the buckets can be retrieved
+      # with a normal call to redis.
+      class BucketStorage
+        # If we have not read buckets for a long time, we might deal with lots
+        # of keys in the union operation. This is why we define a constant that
+        # limits the number of buckets that we send to the union op.
+        #
+        # Currently, we are running a Kinesis job every 2 min and the buckets
+        # are being created every 10s. We could set the constant to 12
+        # (120/10 = 12), but to be sure that we will call union just once on
+        # each job, we are going to set it to 15.
+        MAX_BUCKETS_REDIS_UNION = 15
+        private_constant :MAX_BUCKETS_REDIS_UNION
+
+        attr_reader :storage
+
+        def initialize(storage)
+          @storage = storage
+        end
+
+        # For each of the buckets in the range, deletes it from the set, and
+        # also deletes its contents.
+        def delete_range(last_bucket)
+          buckets = storage.zrangebyscore(Keys.changed_keys_key, 0, last_bucket)
+
+          storage.pipelined do
+            buckets.each { |bucket| delete_bucket_content(bucket) }
+            storage.zremrangebyscore(Keys.changed_keys_key, 0, last_bucket)
+          end
+        end
+
+        def delete_all_buckets_and_keys(options = {})
+          Storage.disable!
+
+          buckets.each do |bucket|
+            keys = storage.smembers(Keys.changed_keys_bucket_key(bucket))
+            unless options[:silent]
+              puts "Deleting bucket: #{bucket}, containing #{keys.size} keys"
+            end
+            storage.del(Keys.changed_keys_bucket_key(bucket))
+          end
+          storage.del(Keys.changed_keys_key)
+        end
+
+        def buckets(first: '-inf', last: '+inf')
+          storage.zrangebyscore(Keys.changed_keys_key, first, last)
+        end
+
+        def pending_buckets_size
+          storage.zcard(Keys.changed_keys_key)
+        end
+
+        # Puts keys in a bucket. The bucket is created if it does not exist.
+        # We could have decided to only fill the bucket if it existed, but that
+        # would affect performance, because we would need to get all the
+        # existing buckets to check if the given one exists in every call.
+        def put_in_bucket(event_keys, bucket)
+          storage.pipelined do
+            storage.zadd(Keys.changed_keys_key, bucket, bucket)
+            storage.sadd(Keys.changed_keys_bucket_key(bucket), event_keys)
+          end
+        end
+
+        def content(buckets)
+          # Values are stored as strings in Redis, but we want integers.
+          # There are some values that can be nil. This happens when the key
+          # has a TTL and we read it once it has expired. Right now, event keys
+          # with granularity = 'minute' expire after 180 s. We might need to
+          # increase that to make sure that we do not miss any values.
+
+          buckets.each_slice(MAX_BUCKETS_REDIS_UNION).inject([]) do |res, buckets_slice|
+            bucket_keys = buckets_slice.map do |bucket|
+              Keys.changed_keys_bucket_key(bucket)
+            end
+            (res + storage.sunion(bucket_keys))
+          end.uniq
+        end
+
+        def pending_keys_by_bucket
+          bucket_keys = buckets.map do |bucket|
+            Keys.changed_keys_bucket_key(bucket)
+          end
+
+          cardinalities = storage.pipelined do
+            bucket_keys.map { |bucket_key| storage.scard(bucket_key) }
+          end
+
+          Hash[buckets.zip(cardinalities)]
+        end
+
+        private
+
+        def delete_bucket_content(bucket)
+          storage.del(Keys.changed_keys_bucket_key(bucket))
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/cleaner.rb

@@ -0,0 +1,195 @@
+module ThreeScale
+  module Backend
+    module Stats
+      class Cleaner
+        # Design notes:
+        # Apisonator does not store in any Redis structure the stats keys
+        # associated with a service. Doing so would imply:
+        #   - Performance hit when reporting. After updating a stats key, it
+        #   would need to be included in a set, which would increase the number
+        #   of operations in Redis.
+        #   - More space in Redis. To maintain the sets mentioned in the point
+        #   above.
+        #   - A data migration would be needed to create those sets from the
+        #   existing stats keys.
+        #
+        # In order to avoid those costs, this class is implemented in a way that
+        # does not need to keep an updated list of all the stats keys for every
+        # service. Services marked for deletion are stored in a Redis set, and
+        # then, a script, periodic cron, etc. is responsible for calling
+        # delete!(). That method scans the whole database and deletes all the
+        # stats keys that belong to those services marked to be deleted.
+        # The downside of this method is that it requires direct access to the
+        # redis servers. Going through a proxy like Twemproxy does not work,
+        # because it does not support the "SCAN" command.
+        #
+        # In the past we tried an alternative approach. When we received a
+        # request to delete the stats of a service, we generated all the
+        # possible stats keys that could exist for it. That approach was not
+        # efficient because it ended up generating many keys that didn't exist
+        # and thus, unnecessary delete calls to Redis. That approach is also
+        # more complex and error prone. You can find the details here:
+        # https://github.com/3scale/apisonator/issues/90
+
+        include Storable
+
+        KEY_SERVICES_TO_DELETE = 'set_with_services_marked_for_deletion'.freeze
+        private_constant :KEY_SERVICES_TO_DELETE
+
+        SLEEP_BETWEEN_SCANS = 0.01 # In seconds
+        private_constant :SLEEP_BETWEEN_SCANS
+
+        SCAN_SLICE = 500
+        private_constant :SCAN_SLICE
+
+        STATS_KEY_PREFIX = 'stats/'.freeze
+        private_constant :STATS_KEY_PREFIX
+
+        class << self
+          include Logging
+          def mark_service_to_be_deleted(service_id)
+            storage.sadd(KEY_SERVICES_TO_DELETE, service_id)
+          end
+
+          # Deletes all the stats for the services that have been marked for
+          # deletion.
+          #
+          # This method receives a collection of instantiated Redis clients.
+          # Those clients need to connect to Redis servers directly. They cannot
+          # connect to a proxy like Twemproxy. The reason is that this function
+          # needs to scan the database using the "SCAN" command, which is not
+          # supported by Twemproxy.
+          #
+          # The services marked as deletion will be marked as done only when
+          # this function finishes deleting the keys from all the Redis servers.
+          # This means that if the function raises in the middle of the
+          # execution, those services will be retried in the next call.
+          #
+          # Note 1: keys deleted cannot be restored.
+          # Note 2: this method can take a long time to finish as it needs to
+          # scan all the keys in several DBs.
+          #
+          # @param [Array] redis_conns Instantiated Redis clients.
+          # @param [IO] log_deleted_keys IO where to write the logs. Defaults to
+          #             nil (logs nothing).
+          def delete!(redis_conns, log_deleted_keys: nil)
+            services = services_to_delete
+            logger.info("Going to delete the stats keys for these services: #{services.to_a}")
+
+            unless services.empty?
+              delete_successful = true
+              redis_conns.each do |redis_conn|
+                begin
+                  delete_keys(redis_conn, services, log_deleted_keys)
+                # If it's a connection error, mark as failed and continue
+                # cleaning other shards. If it's another kind of error, it
+                # could be a bug, so better re-raise.
+                rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
+                  logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
+                  delete_successful = false
+                rescue Redis::CommandError => e
+                  # Redis::CommandError from redis-rb can be raised for multiple
+                  # reasons, so we need to check the error message to distinguish
+                  # connection errors from the rest.
+                  if e.message == 'ERR Connection timed out'.freeze
+                    logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
+                    delete_successful = false
+                  else
+                    raise e
+                  end
+                end
+              end
+
+              remove_services_from_delete_set(services) if delete_successful
+            end
+
+            logger.info("Finished deleting the stats keys for these services: #{services.to_a}")
+          end
+
+          private
+
+          # Returns a set with the services included in the
+          # SET_WITH_SERVICES_MARKED_FOR_DELETION Redis set.
+          def services_to_delete
+            res = []
+            cursor = 0
+
+            loop do
+              cursor, services = storage.sscan(
+                KEY_SERVICES_TO_DELETE, cursor, count: SCAN_SLICE
+              )
+
+              res += services
+
+              break if cursor.to_i == 0
+
+              sleep(SLEEP_BETWEEN_SCANS)
+            end
+
+            res.to_set
+          end
+
+          def delete_keys(redis_conn, services, log_deleted_keys)
+            cursor = 0
+
+            loop do
+              cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
+
+              to_delete = keys.select { |key| delete_key?(key, services) }
+
+              unless to_delete.empty?
+                if log_deleted_keys
+                  values = redis_conn.mget(*(to_delete.to_a))
+                  to_delete.each_with_index do |k, i|
+                    log_deleted_keys.puts "#{k} #{values[i]}"
+                  end
+                end
+
+                redis_conn.del(to_delete)
+              end
+
+              break if cursor.to_i == 0
+
+              sleep(SLEEP_BETWEEN_SCANS)
+            end
+          end
+
+          def remove_services_from_delete_set(services)
+            storage.pipelined do
+              services.each do |service|
+                storage.srem(KEY_SERVICES_TO_DELETE, service)
+              end
+            end
+          end
+
+          def delete_key?(key, services_to_delete)
+            return false unless is_stats_key?(key)
+
+            service_in_key = service_from_stats_key(key)
+            service_in_key && services_to_delete.include?(service_in_key)
+          end
+
+          def is_stats_key?(key)
+            # A key that starts with STATS_KEY_PREFIX is a stats key except if it
+            # follows this pattern: /STATS_KEY_PREFIX{service:.*}\/cinstances/. That's a
+            # type of key used only for the "first traffic" event
+            # (ApplicationEvents.first_traffic).
+            key.start_with?(STATS_KEY_PREFIX) && !key.match(/cinstances/)
+          end
+
+          # Returns nil when there's not a service encoded in the key or when
+          # the stats key has an invalid format.
+          def service_from_stats_key(stats_key)
+            StatsParser.parse(stats_key, nil)[:service]
+          rescue StatsParser::StatsKeyValueInvalid
+            # This could happen with legacy stats keys. For example, a long time
+            # ago some stats keys had a "city" and a "country" encoded, but
+            # always empty. That format has not been used in a long time. We'll
+            # simply ignore those keys.
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/codes_commons.rb

@@ -0,0 +1,14 @@
+module ThreeScale
+  module Backend
+    module Stats
+      module CodesCommons
+        TRACKED_CODES = [200, 404, 403, 500, 503].freeze
+        TRACKED_CODE_GROUPS = ['2XX'.freeze, '4XX'.freeze, '5XX'.freeze].freeze
+
+        def self.get_http_code_group(http_code)
+          "#{http_code / 100}XX"
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/delete_job_def.rb

@@ -0,0 +1,60 @@
+module ThreeScale
+  module Backend
+    module Stats
+      class DeleteJobDef
+        ATTRIBUTES = %i[service_id applications metrics from to context_info].freeze
+        private_constant :ATTRIBUTES
+        attr_reader(*ATTRIBUTES)
+
+        def self.attribute_names
+          ATTRIBUTES
+        end
+
+        def initialize(params = {})
+          ATTRIBUTES.each do |key|
+            instance_variable_set("@#{key}".to_sym, params[key]) unless params[key].nil?
+          end
+          validate
+        end
+
+        def run_async
+          Resque.enqueue(PartitionGeneratorJob, Time.now.getutc.to_f, service_id, applications,
+                         metrics, from, to, context_info)
+        end
+
+        def to_json
+          to_hash.to_json
+        end
+
+        def to_hash
+          Hash[ATTRIBUTES.collect { |key| [key, send(key)] }]
+        end
+
+        private
+
+        def validate
+          # from and to valid epoch times
+          raise_validation_error('from field not integer') unless from.is_a? Integer
+          raise_validation_error('from field is zero') if from.zero?
+          raise_validation_error('to field not integer') unless to.is_a? Integer
+          raise_validation_error('to field is zero') if to.zero?
+          raise_validation_error('from < to fields') if Time.at(to) < Time.at(from)
+          # application is array
+          raise_validation_error('applications field') unless applications.is_a? Array
+          raise_validation_error('applications values') unless applications.all? do |x|
+            x.is_a?(String) || x.is_a?(Integer)
+          end
+          # metrics is array
+          raise_validation_error('metrics field') unless metrics.is_a? Array
+          raise_validation_error('metrics values') unless metrics.all? do |x|
+            x.is_a?(String) || x.is_a?(Integer)
+          end
+        end
+
+        def raise_validation_error(msg)
+          raise DeleteServiceStatsValidationError.new(service_id, msg)
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/stats/key_generator.rb

@@ -0,0 +1,73 @@
+module ThreeScale
+  module Backend
+    module Stats
+      class KeyGenerator
+        attr_reader :service_id, :applications, :metrics, :from, :to
+
+        def initialize(service_id:, applications: [], metrics: [], from:, to:, **)
+          @service_id = service_id
+          @applications = applications
+          @metrics = metrics
+          @from = from
+          @to = to
+        end
+
+        def keys
+          response_code_service_keys +
+            response_code_application_keys +
+            usage_service_keys +
+            usage_application_keys
+        end
+
+        private
+
+        def periods(granularities)
+          granularities.flat_map do |granularity|
+            (Period[granularity].new(Time.at(from))..Period[granularity].new(Time.at(to))).to_a
+          end
+        end
+
+        def response_codes
+          CodesCommons::TRACKED_CODES + CodesCommons::TRACKED_CODE_GROUPS
+        end
+
+        def response_code_service_keys
+          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
+            response_codes.flat_map do |response_code|
+              Keys.service_response_code_value_key(service_id, response_code, period)
+            end
+          end
+        end
+
+        def response_code_application_keys
+          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
+            response_codes.flat_map do |response_code|
+              applications.flat_map do |application|
+                Keys.application_response_code_value_key(service_id, application,
+                                                         response_code, period)
+              end
+            end
+          end
+        end
+
+        def usage_service_keys
+          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
+            metrics.flat_map do |metric|
+              Keys.service_usage_value_key(service_id, metric, period)
+            end
+          end
+        end
+
+        def usage_application_keys
+          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
+            metrics.flat_map do |metric|
+              applications.flat_map do |application|
+                Keys.application_usage_value_key(service_id, application, metric, period)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end