apisonator 3.0.1.1 → 3.3.1

data/lib/3scale/backend/listener_metrics.rb

@@ -4,14 +4,36 @@ require 'rack'
 module ThreeScale
   module Backend
     class ListenerMetrics
-      REQUEST_TYPES = {
+      AUTH_AND_REPORT_REQUEST_TYPES = {
         '/transactions/authorize.xml' => 'authorize',
         '/transactions/oauth_authorize.xml' => 'authorize_oauth',
         '/transactions/authrep.xml' => 'authrep',
         '/transactions/oauth_authrep.xml' => 'authrep_oauth',
         '/transactions.xml' => 'report'
       }
-      private_constant :REQUEST_TYPES
+      private_constant :AUTH_AND_REPORT_REQUEST_TYPES
+
+      # Only the first match is taken into account, that's why for example,
+      # "/\/services\/.*\/stats/" needs to appear before "/\/services/"
+      INTERNAL_API_PATHS = [
+        [/\/services\/.*\/alert_limits/, 'alerts'.freeze],
+        [/\/services\/.*\/applications\/.*\/keys/, 'application_keys'.freeze],
+        [/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'.freeze],
+        [/\/services\/.*\/applications\/.*\/utilization/, 'utilization'.freeze],
+        [/\/services\/.*\/applications/, 'applications'.freeze],
+        [/\/services\/.*\/errors/, 'errors'.freeze],
+        [/\/events/, 'events'.freeze],
+        [/\/services\/.*\/metrics/, 'metrics'.freeze],
+        [/\/service_tokens/, 'service_tokens'.freeze],
+        [/\/services\/.*\/stats/, 'stats'.freeze],
+        [/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'.freeze],
+        [/\/services/, 'services'.freeze],
+      ].freeze
+      private_constant :INTERNAL_API_PATHS
+
+      # Most requests will be under 100ms, so use a higher granularity from there
+      TIME_BUCKETS = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+      private_constant :TIME_BUCKETS
 
       class << self
         ERRORS_4XX_TO_TRACK = Set[403, 404, 409].freeze
@@ -27,9 +49,12 @@ module ThreeScale
         end
 
         def report_resp_code(path, resp_code)
-          Yabeda.apisonator_listener.response_codes.increment(
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_codes.increment(
             {
-              request_type: REQUEST_TYPES[path],
+              request_type: req_type,
               resp_code: code_group(resp_code)
             },
             by: 1
@@ -37,8 +62,11 @@ module ThreeScale
         end
 
         def report_response_time(path, request_time)
-          Yabeda.apisonator_listener.response_times.measure(
-            { request_type: REQUEST_TYPES[path] },
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_times.measure(
+            { request_type: req_type },
             request_time
           )
         end
@@ -69,8 +97,21 @@ module ThreeScale
                 comment 'Response times'
                 unit :seconds
                 tags %i[request_type]
-                # Most requests will be under 100ms, so use a higher granularity from there
-                buckets [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+                buckets TIME_BUCKETS
+              end
+            end
+
+            group :apisonator_listener_internal_api do
+              counter :response_codes do
+                comment 'Response codes'
+                tags %i[request_type resp_code]
+              end
+
+              histogram :response_times do
+                comment 'Response times'
+                unit :seconds
+                tags %i[request_type]
+                buckets TIME_BUCKETS
               end
             end
           end
@@ -93,6 +134,24 @@ module ThreeScale
             'unknown'.freeze
           end
         end
+
+        def req_type(path)
+          AUTH_AND_REPORT_REQUEST_TYPES[path] || internal_api_req_type(path)
+        end
+
+        def internal_api_req_type(path)
+          (_regex, type) = INTERNAL_API_PATHS.find { |(regex, _)| regex.match path }
+          type
+        end
+
+        # Returns the group as defined in .define_metrics
+        def prometheus_group(request_type)
+          if AUTH_AND_REPORT_REQUEST_TYPES.values.include? request_type
+            :apisonator_listener
+          else
+            :apisonator_listener_internal_api
+          end
+        end
       end
     end
   end

data/lib/3scale/backend/stats.rb

@@ -1,8 +1,4 @@
 require '3scale/backend/stats/codes_commons'
 require '3scale/backend/stats/period_commons'
 require '3scale/backend/stats/aggregator'
-require '3scale/backend/stats/delete_job_def'
-require '3scale/backend/stats/key_generator'
-require '3scale/backend/stats/partition_generator_job'
-require '3scale/backend/stats/partition_eraser_job'
 require '3scale/backend/stats/cleaner'

data/lib/3scale/backend/stats/aggregators/base.rb

@@ -20,7 +20,14 @@ module ThreeScale
                 key = counter_key(prefix_key, granularity.new(timestamp))
                 expire_time = Stats::PeriodCommons.expire_time_for_granularity(granularity)
 
-                store_key(cmd, key, value, expire_time)
+                # We don't need to store stats keys set to 0. It wastes Redis
+                # memory because for rate-limiting and stats, a key of set to 0
+                # is equivalent to a key that does not exist.
+                if cmd == :set && value == 0
+                  storage.del(key)
+                else
+                  store_key(cmd, key, value, expire_time)
+                end
 
                 unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity)
                   keys_for_bucket << key

data/lib/3scale/backend/stats/cleaner.rb

@@ -45,6 +45,12 @@ module ThreeScale
         STATS_KEY_PREFIX = 'stats/'.freeze
         private_constant :STATS_KEY_PREFIX
 
+        REDIS_CONN_ERRORS = [Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE].freeze
+        private_constant :REDIS_CONN_ERRORS
+
+        MAX_RETRIES_REDIS_ERRORS = 3
+        private_constant :MAX_RETRIES_REDIS_ERRORS
+
         class << self
           include Logging
           def mark_service_to_be_deleted(service_id)
@@ -77,37 +83,73 @@ module ThreeScale
             logger.info("Going to delete the stats keys for these services: #{services.to_a}")
 
             unless services.empty?
-              delete_successful = true
-              redis_conns.each do |redis_conn|
+              _ok, failed = redis_conns.partition do |redis_conn|
                 begin
                   delete_keys(redis_conn, services, log_deleted_keys)
-                # If it's a connection error, mark as failed and continue
-                # cleaning other shards. If it's another kind of error, it
-                # could be a bug, so better re-raise.
-                rescue Redis::BaseConnectionError, Errno::ECONNREFUSED, Errno::EPIPE => e
-                  logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
-                  delete_successful = false
-                rescue Redis::CommandError => e
-                  # Redis::CommandError from redis-rb can be raised for multiple
-                  # reasons, so we need to check the error message to distinguish
-                  # connection errors from the rest.
-                  if e.message == 'ERR Connection timed out'.freeze
-                    logger.error("Error while deleting stats of server #{redis_conn}: #{e}")
-                    delete_successful = false
-                  else
-                    raise e
-                  end
+                  true
+                rescue => e
+                  handle_redis_exception(e, redis_conn)
+                  false
                 end
               end
 
-              remove_services_from_delete_set(services) if delete_successful
+              with_retries { remove_services_from_delete_set(services) } if failed.empty?
+
+              failed.each do |failed_conn|
+                logger.error("Error while deleting stats of server #{failed_conn}")
+              end
             end
 
             logger.info("Finished deleting the stats keys for these services: #{services.to_a}")
           end
 
+          # Deletes all the stats keys set to 0.
+          #
+          # Stats keys set to 0 are useless and occupy Redis memory
+          # unnecessarily. They were generated due to a bug in previous versions
+          # of Apisonator.
+          # Ref: https://github.com/3scale/apisonator/pull/247
+          #
+          # As the .delete function, this one also receives a collection of
+          # instantiated Redis clients and those need to connect to Redis
+          # servers directly.
+          #
+          # @param [Array] redis_conns Instantiated Redis clients.
+          # @param [IO] log_deleted_keys IO where to write the logs. Defaults to
+          #             nil (logs nothing).
+          def delete_stats_keys_set_to_0(redis_conns, log_deleted_keys: nil)
+            _ok, failed = redis_conns.partition do |redis_conn|
+              begin
+                delete_stats_keys_with_val_0(redis_conn, log_deleted_keys)
+                true
+              rescue => e
+                handle_redis_exception(e, redis_conn)
+                false
+              end
+            end
+
+            failed.each do |failed_conn|
+              logger.error("Error while deleting stats of server #{failed_conn}")
+            end
+          end
+
           private
 
+          def handle_redis_exception(exception, redis_conn)
+            # If it's a connection error, do nothing so we can continue with
+            # other shards. If it's another kind of error, it could be caused by
+            # a bug, so better re-raise.
+
+            case exception
+            when *REDIS_CONN_ERRORS
+              # Do nothing.
+            when Redis::CommandError
+              raise exception if exception.message != 'ERR Connection timed out'.freeze
+            else
+              raise exception
+            end
+          end
+
           # Returns a set with the services included in the
           # SET_WITH_SERVICES_MARKED_FOR_DELETION Redis set.
           def services_to_delete
@@ -133,19 +175,21 @@ module ThreeScale
             cursor = 0
 
             loop do
-              cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
+              with_retries do
+                cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
 
-              to_delete = keys.select { |key| delete_key?(key, services) }
+                to_delete = keys.select { |key| delete_key?(key, services) }
 
-              unless to_delete.empty?
-                if log_deleted_keys
-                  values = redis_conn.mget(*(to_delete.to_a))
-                  to_delete.each_with_index do |k, i|
-                    log_deleted_keys.puts "#{k} #{values[i]}"
+                unless to_delete.empty?
+                  if log_deleted_keys
+                    values = redis_conn.mget(*(to_delete.to_a))
+                    to_delete.each_with_index do |k, i|
+                      log_deleted_keys.puts "#{k} #{values[i]}"
+                    end
                   end
-                end
 
-                redis_conn.del(to_delete)
+                  redis_conn.del(to_delete)
+                end
               end
 
               break if cursor.to_i == 0
@@ -188,6 +232,43 @@ module ThreeScale
             # simply ignore those keys.
             nil
           end
+
+          def delete_stats_keys_with_val_0(redis_conn, log_deleted_keys)
+            cursor = 0
+
+            loop do
+              with_retries do
+                cursor, keys = redis_conn.scan(cursor, count: SCAN_SLICE)
+
+                stats_keys = keys.select { |k| is_stats_key?(k) }
+
+                unless stats_keys.empty?
+                  values = redis_conn.mget(*stats_keys)
+                  to_delete = stats_keys.zip(values).select { |_, v| v == '0'.freeze }.map(&:first)
+
+                  unless to_delete.empty?
+                    redis_conn.del(to_delete)
+                    to_delete.each { |k| log_deleted_keys.puts k } if log_deleted_keys
+                  end
+                end
+              end
+
+              break if cursor.to_i == 0
+
+              sleep(SLEEP_BETWEEN_SCANS)
+            end
+          end
+
+          def with_retries(max = MAX_RETRIES_REDIS_ERRORS)
+            retries = 0
+            begin
+              yield
+            rescue Exception => e
+              retries += 1
+              retry if retries < max
+              raise e
+            end
+          end
         end
       end
     end

data/lib/3scale/backend/stats/keys.rb

@@ -70,6 +70,12 @@ module ThreeScale
           key
         end
 
+        def set_of_apps_with_traffic(service_id)
+          Stats::Keys.applications_key_prefix(
+            Stats::Keys.service_key_prefix(service_id)
+          )
+        end
+
         # We want all the buckets to go to the same Redis shard.
         # The reason is that SUNION support in Twemproxy requires that the
         # supplied keys hash to the same server.

data/lib/3scale/backend/stats/period_commons.rb

@@ -12,9 +12,6 @@ module ThreeScale
         GRANULARITY_EXPIRATION_TIME = { Period[:minute] => 180 }.freeze
         private_constant :GRANULARITY_EXPIRATION_TIME
 
-        PERMANENT_SERVICE_GRANULARITIES = (SERVICE_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-        PERMANENT_EXPANDED_GRANULARITIES = (EXPANDED_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-
         # We are not going to send metrics with granularity 'eternity' or
         # 'week' to Kinesis, so there is no point in storing them in Redis
         # buckets.

data/lib/3scale/backend/transactor.rb

@@ -20,8 +20,14 @@ module ThreeScale
       def report(provider_key, service_id, transactions, context_info = {})
         service = Service.load_with_provider_key!(service_id, provider_key)
 
-        report_enqueue(service.id, transactions, context_info)
-        notify_report(provider_key, transactions.size)
+        # A usage of 0 does not affect rate-limits or stats, so we do not need
+        # to report it.
+        filtered_transactions = filter_usages_with_0(transactions.clone)
+
+        return if filtered_transactions.empty?
+
+        report_enqueue(service.id, filtered_transactions, context_info)
+        notify_report(provider_key, filtered_transactions.size)
       end
 
       def authorize(provider_key, params, context_info = {})
@@ -137,9 +143,17 @@ module ThreeScale
 
         usage = params[:usage]
 
-        if (usage || params[:log]) && status.authorized?
+        filtered_usage = filter_metrics_without_inc(usage.clone) if usage
+
+        if ((filtered_usage && !filtered_usage.empty?) || params[:log]) && status.authorized?
           application_id = status.application.id
-          report_enqueue(status.service_id, { 0 => {"app_id" => application_id, "usage" => usage, "log" => params[:log] } }, request: { extensions: request_info[:extensions] })
+
+          report_enqueue(
+            status.service_id,
+            { 0 => {"app_id" => application_id, "usage" => filtered_usage, "log" => params[:log] } },
+            request: { extensions: request_info[:extensions] }
+          )
+
           notify_authrep(provider_key, usage ? 1 : 0)
         else
           notify_authorize(provider_key)
@@ -182,6 +196,19 @@ module ThreeScale
         end
       end
 
+      def filter_usages_with_0(transactions)
+        # There are plenty of existing tests using both a string and a symbol
+        # when accessing the usage.
+        transactions.delete_if do |_idx, tx|
+          (usage = tx['usage'.freeze] || tx[:usage]) or next
+          filter_metrics_without_inc(usage).empty?
+        end
+      end
+
+      def filter_metrics_without_inc(usage)
+        usage.delete_if { |_metric, delta| delta.to_s == '0'.freeze }
+      end
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/transactor/notify_batcher.rb

@@ -30,9 +30,13 @@ module ThreeScale
         end
 
         def notify(provider_key, usage)
-          # batch several notifications together so that we can process just one
+          # We need the master service ID to report its metrics. If it's not
+          # set, we don't need to notify anything.
+          # Batch several notifications together so that we can process just one
           # job for a group of them.
-          notify_batch(provider_key, usage)
+          unless configuration.master_service_id.to_s.empty?
+            notify_batch(provider_key, usage)
+          end
         end
 
         def notify_batch(provider_key, usage)

data/lib/3scale/backend/transactor/notify_job.rb

@@ -7,8 +7,6 @@ module ThreeScale
         extend Configurable
         @queue = :main
 
-        InvalidMasterServiceId = Class.new(ThreeScale::Backend::Error)
-
         class << self
           def perform_logged(provider_key, usage, timestamp, _enqueue_time)
             application_id = Application.load_id_by_key(master_service_id, provider_key)
@@ -16,12 +14,42 @@ module ThreeScale
             if application_id && Application.exists?(master_service_id, application_id)
               master_metrics = Metric.load_all(master_service_id)
 
-              ProcessJob.perform([{
-                service_id: master_service_id,
-                application_id: application_id,
-                timestamp: timestamp,
-                usage: master_metrics.process_usage(usage)
-              }])
+              begin
+                ProcessJob.perform([{
+                  service_id: master_service_id,
+                  application_id: application_id,
+                  timestamp: timestamp,
+                  usage: master_metrics.process_usage(usage)
+                }])
+              rescue MetricInvalid => e
+                # This happens when the master account in Porta does not have
+                # the notify metrics defined (by default "transactions" and
+                # "transactions/authorize"). These metrics need to be created in
+                # Porta, Apisonator does not have a way to guarantee that
+                # they're defined.
+                # Notice that this rescue prevents the job from being retried.
+                # Apisonator can't know when the metrics will be created (if
+                # ever) so it's better to log the error rather than retrying
+                # these jobs for an undefined period of time.
+                Worker.logger.notify(e)
+                return [false, "#{e}"]
+              rescue TransactionTimestampNotWithinRange => e
+                # This is very unlikely to happen. The timestamps in a notify
+                # job are not set by users, they are set by the listeners. If
+                # this error happens it might mean that:
+                # a) The worker started processing this job way after the
+                # listener produced it. This can happen for example if we make
+                # some requests to a listener with no workers. The listeners
+                # will enqueue some notify jobs. If we start a worker hours
+                # later, we might see this error.
+                # b) There's some kind of clock skew issue.
+                # c) There's a bug.
+                #
+                # We can't raise here, because then, the job will be retried,
+                # but it's going to fail always if it has an old timestamp.
+                Worker.logger.notify(e)
+                return [false, "#{provider_key} #{application_id} #{e}"]
+              end
             end
             [true, "#{provider_key} #{application_id || '--'}"]
           end
@@ -29,15 +57,7 @@ module ThreeScale
           private
 
           def master_service_id
-            value = configuration.master_service_id
-
-            unless value
-              raise InvalidMasterServiceId,
-                    "Can't find master service id. Make sure the \"master_service_id\" "\
-                    'configuration value is set correctly'
-            end
-
-            value.to_s
+            configuration.master_service_id.to_s
           end
         end
       end