apisonator 3.2.1 → 3.3.3

data/app/api/internal/stats.rb

@@ -6,32 +6,13 @@ module ThreeScale
           respond_with_404('service not found') unless Service.exists?(params[:service_id])
         end
 
-        # This is very slow and needs to be disabled until the performance
-        # issues are solved. In the meanwhile, the job will just return OK.
-=begin
-        delete '' do |service_id|
-          delete_stats_job_attrs = api_params Stats::DeleteJobDef
-          delete_stats_job_attrs[:service_id] = service_id
-          delete_stats_job_attrs[:from] = delete_stats_job_attrs[:from].to_i
-          delete_stats_job_attrs[:to] = delete_stats_job_attrs[:to].to_i
-          begin
-            Stats::DeleteJobDef.new(delete_stats_job_attrs).run_async
-          rescue DeleteServiceStatsValidationError => e
-            [400, headers, { status: :error, error: e.message }.to_json]
-          else
-            { status: :to_be_deleted }.to_json
-          end
-=end
-
-        # This is an alternative to the above. It just adds the service to a
-        # Redis set to marked is as "to be deleted".
-        # Later a script can read that set and actually delete the keys.
-        # Read the docs of the Stats::Cleaner class for more details.
+        # This adds the service to a Redis set to mark is as "to be deleted".
+        # Later a script can read that set and actually delete the keys. Read
+        # the docs of the Stats::Cleaner class for more details.
         #
-        # Notice that this method ignores the "from" and "to" parameters. When
-        # system calls this method, they're always interested in deleting all
-        # the keys. They were just passing "from" and "to" to make the
-        # implementation of the option above easier.
+        # Notice that this method ignores the "from" and "to" parameters used in
+        # previous versions. When system calls this method, they're always
+        # interested in deleting all the keys.
         delete '' do |service_id|
           Stats::Cleaner.mark_service_to_be_deleted(service_id)
           { status: :to_be_deleted }.to_json

data/lib/3scale/backend/alerts.rb

@@ -6,11 +6,10 @@ module ThreeScale
 
         # The compacted hour in the params refers to the
         # TimeHacks.to_compact_s method.
-        def alert_keys(service_id, app_id, discrete_utilization, compacted_hour_start)
+        def alert_keys(service_id, app_id, discrete_utilization)
           {
             already_notified: key_already_notified(service_id, app_id, discrete_utilization),
             allowed: key_allowed_set(service_id),
-            current_max: key_current_max(service_id, app_id, compacted_hour_start),
             current_id: key_current_id
           }
         end
@@ -31,11 +30,6 @@ module ThreeScale
           "#{prefix}allowed_set"
         end
 
-        def key_current_max(service_id, app_id, compacted_hour_start)
-          prefix = key_prefix(service_id, app_id)
-          "#{prefix}#{compacted_hour_start}/current_max"
-        end
-
         def key_current_id
           'alerts/current_id'.freeze
         end
@@ -43,6 +37,7 @@ module ThreeScale
 
       extend self
       extend KeyHelpers
+      include Memoizer::Decorator
 
       ALERT_TTL       = 24*3600 # 1 day (only one message per day)
       ## zero must be here and sorted, yes or yes
@@ -50,6 +45,16 @@ module ThreeScale
       FIRST_ALERT_BIN = ALERT_BINS.first
       RALERT_BINS     = ALERT_BINS.reverse.freeze
 
+      def can_raise_more_alerts?(service_id, app_id)
+        allowed_bins = allowed_set_for_service(service_id).sort
+
+        return false if allowed_bins.empty?
+
+        # If the bin with the highest value has already been notified, there's
+        # no need to notify anything else.
+        not notified?(service_id, app_id, allowed_bins.last)
+      end
+
       def utilization(app_usage_reports)
         max_utilization = -1.0
         max_record = nil
@@ -77,25 +82,12 @@ module ThreeScale
 
       def update_utilization(service_id, app_id, max_utilization, max_record, timestamp)
         discrete = utilization_discrete(max_utilization)
-        max_utilization_i = (max_utilization * 100.0).round
 
-        beginning_of_day = Period::Boundary.day_start(timestamp)
-        period_hour = Period::Boundary.hour_start(timestamp).to_compact_s
-        # UNIX timestamp for key expiration - add 1 day + 5 mins
-        expire_at = (beginning_of_day + 86700).to_i
+        keys = alert_keys(service_id, app_id, discrete)
 
-        keys = alert_keys(service_id, app_id, discrete, period_hour)
-
-        already_alerted, allowed, current_max, _ = storage.pipelined do
+        already_alerted, allowed = storage.pipelined do
           storage.get(keys[:already_notified])
           storage.sismember(keys[:allowed], discrete)
-          storage.get(keys[:current_max])
-          storage.expireat(keys[:current_max], expire_at)
-        end
-
-        ## update the status of utilization
-        if max_utilization_i > current_max.to_i
-          storage.set(keys[:current_max], max_utilization_i)
         end
 
         if already_alerted.nil? && allowed && discrete.to_i > 0
@@ -129,6 +121,16 @@ module ThreeScale
         "#{record.current_value}/#{record.max_value}"
       end
 
+      def allowed_set_for_service(service_id)
+        storage.smembers(key_allowed_set(service_id)).map(&:to_i) # Redis returns strings always
+      end
+      memoize :allowed_set_for_service
+
+      def notified?(service_id, app_id, bin)
+        storage.get(key_already_notified(service_id, app_id, bin))
+      end
+      memoize :notified?
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/application_events.rb

@@ -40,10 +40,8 @@ module ThreeScale
       private
 
       def self.first_traffic(service_id, application_id)
-        key = Stats::Keys.applications_key_prefix(
-          Stats::Keys.service_key_prefix(service_id)
-        )
-        if storage.sadd(key, encode_key(application_id))
+        if storage.sadd(Stats::Keys.set_of_apps_with_traffic(service_id),
+                        encode_key(application_id))
           EventStorage.store(:first_traffic,
                              { service_id:     service_id,
                                application_id: application_id,

data/lib/3scale/backend/configuration.rb

@@ -32,8 +32,6 @@ module ThreeScale
 
     CONFIG_DELETE_STATS_BATCH_SIZE = 50
     private_constant :CONFIG_DELETE_STATS_BATCH_SIZE
-    CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE = 1000
-    private_constant :CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE
 
     @configuration = Configuration::Loader.new
 
@@ -54,13 +52,12 @@ module ThreeScale
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)
       config.add_section(:hoptoad, :service, :api_key)
-      config.add_section(:stats, :bucket_size, :delete_batch_size, :delete_partition_batch_size)
+      config.add_section(:stats, :bucket_size, :delete_batch_size)
       config.add_section(:redshift, :host, :port, :dbname, :user, :password)
       config.add_section(:statsd, :host, :port)
       config.add_section(:internal_api, :user, :password)
       config.add_section(:master, :metrics)
       config.add_section(:worker_prometheus_metrics, :enabled, :port)
-      config.add_section(:listener_prometheus_metrics, :enabled, :port)
 
       config.add_section(
           :async_worker,
@@ -125,9 +122,6 @@ module ThreeScale
       config.stats.delete_batch_size = parse_int(config.stats.delete_batch_size,
                                                  CONFIG_DELETE_STATS_BATCH_SIZE)
 
-      config.stats.delete_partition_batch_size = parse_int(config.stats.delete_partition_batch_size,
-                                                           CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE)
-
       # often we don't have a log_file setting - generate it here from
       # the log_path setting.
       log_file = config.log_file

data/lib/3scale/backend/errors.rb

@@ -292,12 +292,6 @@ module ThreeScale
       end
     end
 
-    class DeleteServiceStatsValidationError < Error
-      def initialize(service_id, msg)
-        super "Delete stats job context validation error. Service: #{service_id}. Error: #{msg}"
-      end
-    end
-
     class EndUsersNoLongerSupported < BadRequest
       def initialize
         super 'End-users are no longer supported, do not specify the user_id parameter'.freeze

data/lib/3scale/backend/job_fetcher.rb

@@ -32,25 +32,6 @@ module ThreeScale
             DEFAULT_WAIT_BEFORE_FETCHING_MORE_JOBS
       end
 
-      def pop_from_queue
-        begin
-          encoded_job = @redis.blpop(*@queues, timeout: @fetch_timeout)
-        rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
-          raise RedisConnectionError.new(e.message)
-        rescue Redis::CommandError => e
-          # Redis::CommandError from redis-rb can be raised for multiple
-          # reasons, so we need to check the error message to distinguish
-          # connection errors from the rest.
-          if e.message == 'ERR Connection timed out'.freeze
-            raise RedisConnectionError.new(e.message)
-          else
-            raise e
-          end
-        end
-
-        encoded_job
-      end
-
       def fetch
         encoded_job = pop_from_queue
         return nil if encoded_job.nil? || encoded_job.empty?
@@ -99,10 +80,11 @@ module ThreeScale
 
               # Re-instantiate Redis instance. This is needed to recover from
               # Errno::EPIPE, not sure if there are others.
-              @redis = ThreeScale::Backend::QueueStorage.connection(
-                  ThreeScale::Backend.environment,
-                  ThreeScale::Backend.configuration
+              @redis = Redis::Namespace.new(
+                WorkerAsync.const_get(:RESQUE_REDIS_NAMESPACE),
+                redis: QueueStorage.connection(Backend.environment, Backend.configuration)
               )
+
              # If there is a different kind of error, it's probably a
              # programming error. Like sending an invalid blpop command to
              # Redis. In that case, let the worker crash.
@@ -111,12 +93,36 @@ module ThreeScale
           end
         end
 
+      rescue Exception => e
+        Worker.logger.notify(e)
+      ensure
         job_queue.close
       end
 
       def shutdown
         @shutdown = true
       end
+
+      private
+
+      def pop_from_queue
+        begin
+          encoded_job = @redis.blpop(*@queues, timeout: @fetch_timeout)
+        rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
+          raise RedisConnectionError.new(e.message)
+        rescue Redis::CommandError => e
+          # Redis::CommandError from redis-rb can be raised for multiple
+          # reasons, so we need to check the error message to distinguish
+          # connection errors from the rest.
+          if e.message == 'ERR Connection timed out'.freeze
+            raise RedisConnectionError.new(e.message)
+          else
+            raise e
+          end
+        end
+
+        encoded_job
+      end
     end
   end
 end

data/lib/3scale/backend/rack.rb

@@ -17,7 +17,10 @@ module ThreeScale
 
           Backend::Logging::External.setup_rack self
 
-          if Backend.configuration.listener_prometheus_metrics.enabled
+          # Notice that this cannot be specified via config, it needs to be an
+          # ENV because the metric server is started in Puma/Falcon
+          # "before_fork" and the configuration is not loaded at that point.
+          if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s.downcase.freeze == 'true'.freeze
             use Rack::Prometheus
           end
 

data/lib/3scale/backend/stats.rb

@@ -1,8 +1,4 @@
 require '3scale/backend/stats/codes_commons'
 require '3scale/backend/stats/period_commons'
 require '3scale/backend/stats/aggregator'
-require '3scale/backend/stats/delete_job_def'
-require '3scale/backend/stats/key_generator'
-require '3scale/backend/stats/partition_generator_job'
-require '3scale/backend/stats/partition_eraser_job'
 require '3scale/backend/stats/cleaner'

data/lib/3scale/backend/stats/aggregator.rb

@@ -145,6 +145,16 @@ module ThreeScale
               application = Backend::Application.load(service_id,
                                                       values[:application_id])
 
+              # The app could have been deleted at some point since the job was
+              # enqueued. No need to update alerts in that case.
+              next unless application
+
+              # The operations below are costly. They load all the usage limits
+              # and current usages to find the current utilization levels.
+              # That's why before that, we check if there are any alerts that
+              # can be raised.
+              next unless Alerts.can_raise_more_alerts?(service_id, values[:application_id])
+
               application.load_metric_names
               usage = Usage.application_usage(application, current_timestamp)
               status = Transactor::Status.new(service_id: service_id,

data/lib/3scale/backend/stats/aggregators/base.rb

@@ -20,7 +20,14 @@ module ThreeScale
                 key = counter_key(prefix_key, granularity.new(timestamp))
                 expire_time = Stats::PeriodCommons.expire_time_for_granularity(granularity)
 
-                store_key(cmd, key, value, expire_time)
+                # We don't need to store stats keys set to 0. It wastes Redis
+                # memory because for rate-limiting and stats, a key of set to 0
+                # is equivalent to a key that does not exist.
+                if cmd == :set && value == 0
+                  storage.del(key)
+                else
+                  store_key(cmd, key, value, expire_time)
+                end
 
                 unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity)
                   keys_for_bucket << key

data/lib/3scale/backend/stats/cleaner.rb

@@ -45,6 +45,12 @@ module ThreeScale
         STATS_KEY_PREFIX = 'stats/'.freeze
         private_constant :STATS_KEY_PREFIX
 
+        REDIS_CONN_ERRORS = [Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE].freeze
+        private_constant :REDIS_CONN_ERRORS
+
+        MAX_RETRIES_REDIS_ERRORS = 3
+        private_constant :MAX_RETRIES_REDIS_ERRORS
+
         class << self
           include Logging
           def mark_service_to_be_deleted(service_id)
@@ -77,37 +83,73 @@ module ThreeScale
             logger.info("Going to delete the stats keys for these services: #{services.to_a}")
 
             unless services.empty?
-              delete_successful = true
-              redis_conns.each do |redis_conn|
+              _ok, failed = redis_conns.partition do |redis_conn|
                 begin
                   delete_keys(redis_conn, services, log_deleted_keys)
-                # If it's a connection error, mark as failed and continue
-                # cleaning other shards. If it's another kind of error, it
-                # could be a bug, so better re-raise.
-                rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
-                  logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
-                  delete_successful = false
-                rescue Redis::CommandError => e
-                  # Redis::CommandError from redis-rb can be raised for multiple
-                  # reasons, so we need to check the error message to distinguish
-                  # connection errors from the rest.
-                  if e.message == 'ERR Connection timed out'.freeze
-                    logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
-                    delete_successful = false
-                  else
-                    raise e
-                  end
+                  true
+                rescue => e
+                  handle_redis_exception(e, redis_conn)
+                  false
                 end
               end
 
-              remove_services_from_delete_set(services) if delete_successful
+              with_retries { remove_services_from_delete_set(services) } if failed.empty?
+
+              failed.each do |failed_conn|
+                logger.error("Error while deleting stats of server #{failed_conn}")
+              end
             end
 
             logger.info("Finished deleting the stats keys for these services: #{services.to_a}")
           end
 
+          # Deletes all the stats keys set to 0.
+          #
+          # Stats keys set to 0 are useless and occupy Redis memory
+          # unnecessarily. They were generated due to a bug in previous versions
+          # of Apisonator.
+          # Ref: https://github.com/3scale/apisonator/pull/247
+          #
+          # As the .delete function, this one also receives a collection of
+          # instantiated Redis clients and those need to connect to Redis
+          # servers directly.
+          #
+          # @param [Array] redis_conns Instantiated Redis clients.
+          # @param [IO] log_deleted_keys IO where to write the logs. Defaults to
+          #             nil (logs nothing).
+          def delete_stats_keys_set_to_0(redis_conns, log_deleted_keys: nil)
+            _ok, failed = redis_conns.partition do |redis_conn|
+              begin
+                delete_stats_keys_with_val_0(redis_conn, log_deleted_keys)
+                true
+              rescue => e
+                handle_redis_exception(e, redis_conn)
+                false
+              end
+            end
+
+            failed.each do |failed_conn|
+              logger.error("Error while deleting stats of server #{failed_conn}")
+            end
+          end
+
           private
 
+          def handle_redis_exception(exception, redis_conn)
+            # If it's a connection error, do nothing so we can continue with
+            # other shards. If it's another kind of error, it could be caused by
+            # a bug, so better re-raise.
+
+            case exception
+            when *REDIS_CONN_ERRORS
+              # Do nothing.
+            when Redis::CommandError
+              raise exception if exception.message != 'ERR Connection timed out'.freeze
+            else
+              raise exception
+            end
+          end
+
           # Returns a set with the services included in the
           # SET_WITH_SERVICES_MARKED_FOR_DELETION Redis set.
           def services_to_delete
@@ -133,19 +175,21 @@ module ThreeScale
             cursor = 0
 
             loop do
-              cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
+              with_retries do
+                cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
 
-              to_delete = keys.select { |key| delete_key?(key, services) }
+                to_delete = keys.select { |key| delete_key?(key, services) }
 
-              unless to_delete.empty?
-                if log_deleted_keys
-                  values = redis_conn.mget(*(to_delete.to_a))
-                  to_delete.each_with_index do |k, i|
-                    log_deleted_keys.puts "#{k} #{values[i]}"
+                unless to_delete.empty?
+                  if log_deleted_keys
+                    values = redis_conn.mget(*(to_delete.to_a))
+                    to_delete.each_with_index do |k, i|
+                      log_deleted_keys.puts "#{k} #{values[i]}"
+                    end
                   end
-                end
 
-                redis_conn.del(to_delete)
+                  redis_conn.del(to_delete)
+                end
               end
 
               break if cursor.to_i == 0
@@ -188,6 +232,43 @@ module ThreeScale
             # simply ignore those keys.
             nil
           end
+
+          def delete_stats_keys_with_val_0(redis_conn, log_deleted_keys)
+            cursor = 0
+
+            loop do
+              with_retries do
+                cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
+
+                stats_keys = keys.select { |k| is_stats_key?(k) }
+
+                unless stats_keys.empty?
+                  values = redis_conn.mget(*stats_keys)
+                  to_delete = stats_keys.zip(values).select { |_, v| v == '0'.freeze }.map(&:first)
+
+                  unless to_delete.empty?
+                    redis_conn.del(to_delete)
+                    to_delete.each { |k| log_deleted_keys.puts k } if log_deleted_keys
+                  end
+                end
+              end
+
+              break if cursor.to_i == 0
+
+              sleep(SLEEP_BETWEEN_SCANS)
+            end
+          end
+
+          def with_retries(max = MAX_RETRIES_REDIS_ERRORS)
+            retries = 0
+            begin
+              yield
+            rescue Exception => e
+              retries += 1
+              retry if retries < max
+              raise e
+            end
+          end
         end
       end
     end