apisonator 3.4.0 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7a260bc57e82268c54885b58a12c79d28c9b7c53f8fb5e48182c887488888184
-  data.tar.gz: 7dbed84c520c06e914d15e99f6e1b78365a1b695d8cf8e1fd66a58c02fa36fa9
+  metadata.gz: 78b328c64420fa58f4480eb39b555856dd9a032ba78eed32f084767988cea59d
+  data.tar.gz: de454b45677bc1c133eeda34c8fb4ebfe56253c4b4de851f1d192652c5baf999
 SHA512:
-  metadata.gz: 4d0bdd90d90972bc416faf04a852ff56c3895da1cb191b264617f71add2f22c5942d928d85c9877faceab4f00e7fe26955dfeed7c6cbc615a8fe555b7c6e251a
-  data.tar.gz: 22b6dbdb74d73117ba7358d231b5f747f54746687e452285b77883df4b352d20d59b3997b38a0c5c7b6cba9bce618b41900d6a0ee3ea1e169727470bce8158cb
+  metadata.gz: bf7613f456c9810217adfbdd5f345b86aa59d9abdccd44d308d846e7f271849d484601900348a4dcd4a16437d9e0376b7927e3675374a9be7a2af723c2820626
+  data.tar.gz: deb0823a0648f55bf24c24da6727afa4e62d2d5d92eeeee5545b9f6c8af38b2105bc8bb8caf3f7b28c187e020080b9b9f0b43721a42dfece2d6963d698a23e9a

data/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.4.1 - 2021-06-16
+
+### Changed
+
+- Introduced performance optimizations around alerts detection
+([#287](https://github.com/3scale/apisonator/pull/287)).
+
 ## 3.4.0 - 2021-06-14
 
 ### Added

data/Gemfile.lock

@@ -36,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.4.0)
+    apisonator (3.4.1)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile.on_prem.lock

@@ -36,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.4.0)
+    apisonator (3.4.1)
 
 GEM
   remote: https://rubygems.org/

data/lib/3scale/backend.rb

@@ -44,6 +44,7 @@ require '3scale/backend/queue_storage'
 require '3scale/backend/errors'
 require '3scale/backend/stats'
 require '3scale/backend/usage_limit'
+require '3scale/backend/utilization'
 require '3scale/backend/alerts'
 require '3scale/backend/event_storage'
 require '3scale/backend/worker'

data/lib/3scale/backend/alert_limit.rb

@@ -9,7 +9,11 @@ module ThreeScale
       attr_accessor :service_id, :value
 
       def save
-        storage.sadd(key_allowed_set(service_id), value.to_i) if valid?
+        if valid?
+          res = storage.sadd(key_allowed_set(service_id), value.to_i)
+          Alerts::UsagesChecked.invalidate_for_service(service_id)
+          res
+        end
       end
 
       def to_hash

data/lib/3scale/backend/alerts.rb

@@ -33,6 +33,11 @@ module ThreeScale
         def key_current_id
           'alerts/current_id'.freeze
         end
+
+        def key_usage_already_checked(service_id, app_id)
+          prefix = key_prefix(service_id, app_id)
+          "#{prefix}usage_already_checked"
+        end
       end
 
       extend self
@@ -45,43 +50,82 @@ module ThreeScale
       FIRST_ALERT_BIN = ALERT_BINS.first
       RALERT_BINS     = ALERT_BINS.reverse.freeze
 
-      def can_raise_more_alerts?(service_id, app_id)
-        allowed_bins = allowed_set_for_service(service_id).sort
+      # This class is useful to reduce the amount of information that we need to
+      # fetch from Redis to determine whether an alert should be raised.
+      # In summary, alerts are raised at the application level and we need to
+      # wait for 24h before raising a new one for the same level (ALERTS_BIN
+      # above).
+      #
+      # This class allows us to check all the usage limits once and then not
+      # check all of them again (just the ones in the report job) until:
+      # 1) A specific alert has expired (24h passed since it was triggered).
+      # 2) A new alert bin is enabled for the service.
+      class UsagesChecked
+        extend KeyHelpers
+        extend StorageHelpers
+        include Memoizer::Decorator
+
+        def self.need_to_check_all?(service_id, app_id)
+          !storage.exists(key_usage_already_checked(service_id, app_id))
+        end
+        memoize :need_to_check_all?
 
-        return false if allowed_bins.empty?
+        def self.mark_all_checked(service_id, app_id)
+          ttl = ALERT_BINS.map do |bin|
+            ttl = storage.ttl(key_already_notified(service_id, app_id, bin))
 
-        # If the bin with the highest value has already been notified, there's
-        # no need to notify anything else.
-        not notified?(service_id, app_id, allowed_bins.last)
-      end
+            # Redis returns -2 if key does not exist, and -1 if it exists without
+            # a TTL (we know this should not happen for the alert bins).
+            # In those cases we should just set the TTL to the max (ALERT_TTL).
+            ttl >= 0 ? ttl : ALERT_TTL
+          end.min
 
-      def utilization(app_usage_reports)
-        max_utilization = -1.0
-        max_record = nil
-        max = proc do |item|
-          if item.max_value > 0
-            utilization = item.current_value / item.max_value.to_f
-
-            if utilization > max_utilization
-              max_record = item
-              max_utilization = utilization
-            end
-          end
+          storage.setex(key_usage_already_checked(service_id, app_id), ttl, '1'.freeze)
+          Memoizer.clear(Memoizer.build_key(self, :need_to_check_all?, service_id, app_id))
         end
 
-        app_usage_reports.each(&max)
+        def self.invalidate(service_id, app_id)
+          storage.del(key_usage_already_checked(service_id, app_id))
+        end
+
+        def self.invalidate_for_service(service_id)
+          app_ids = []
+          cursor = 0
+
+          loop do
+            cursor, ids = storage.sscan(
+              Application.applications_set_key(service_id), cursor, count: SCAN_SLICE
+            )
+
+            app_ids += ids
 
-        if max_utilization == -1
-          ## case that all the limits have max_value==0
-          max_utilization = 0
-          max_record = app_usage_reports.first
+            break if cursor.to_i == 0
+          end
+
+          invalidate_batch(service_id, app_ids)
+        end
+
+        def self.invalidate_batch(service_id, app_ids)
+          app_ids.each_slice(PIPELINED_SLICE_SIZE) do |ids|
+            keys = ids.map { |app_id| key_usage_already_checked(service_id, app_id) }
+            storage.del(keys)
+          end
         end
+        private_class_method :invalidate_batch
+      end
+
+      def can_raise_more_alerts?(service_id, app_id)
+        allowed_bins = allowed_set_for_service(service_id).sort
+
+        return false if allowed_bins.empty?
 
-        [max_utilization, max_record]
+        # If the bin with the highest value has already been notified, there's
+        # no need to notify anything else.
+        not notified?(service_id, app_id, allowed_bins.last)
       end
 
-      def update_utilization(service_id, app_id, max_utilization, max_record, timestamp)
-        discrete = utilization_discrete(max_utilization)
+      def update_utilization(service_id, app_id, utilization)
+        discrete = utilization_discrete(utilization.ratio)
 
         keys = alert_keys(service_id, app_id, discrete)
 
@@ -91,18 +135,19 @@ module ThreeScale
         end
 
         if already_alerted.nil? && allowed && discrete.to_i > 0
-          next_id, _ = storage.pipelined do
+          next_id, _, _ = storage.pipelined do
             storage.incr(keys[:current_id])
             storage.setex(keys[:already_notified], ALERT_TTL, "1")
+            UsagesChecked.invalidate(service_id, app_id)
           end
 
           alert = { :id => next_id,
                     :utilization => discrete,
-                    :max_utilization => max_utilization,
+                    :max_utilization => utilization.ratio,
                     :application_id => app_id,
                     :service_id => service_id,
-                    :timestamp => timestamp,
-                    :limit => formatted_limit(max_record) }
+                    :timestamp => Time.now.utc,
+                    :limit => utilization.to_s }
 
           Backend::EventStorage::store(:alert, alert)
         end
@@ -116,11 +161,6 @@ module ThreeScale
         end || FIRST_ALERT_BIN
       end
 
-      def formatted_limit(record)
-        "#{record.metric_name} per #{record.period}: "\
-        "#{record.current_value}/#{record.max_value}"
-      end
-
       def allowed_set_for_service(service_id)
         storage.smembers(key_allowed_set(service_id)).map(&:to_i) # Redis returns strings always
       end

data/lib/3scale/backend/constants.rb

@@ -4,6 +4,8 @@ module ThreeScale
       module All
         TIME_FORMAT          = '%Y-%m-%d %H:%M:%S %z'.freeze
         PIPELINED_SLICE_SIZE = 400
+        SLEEP_BETWEEN_SCANS = 0.01 # In seconds
+        SCAN_SLICE = 500
       end
 
       def self.included(base)

data/lib/3scale/backend/stats/aggregator.rb

@@ -59,7 +59,7 @@ module ThreeScale
             touched_apps = aggregate(transactions, current_bucket)
 
             ApplicationEvents.generate(touched_apps.values)
-            update_alerts(touched_apps)
+            update_alerts(transactions)
             begin
               ApplicationEvents.ping
             rescue ApplicationEvents::PingFailed => e
@@ -137,39 +137,35 @@ module ThreeScale
             logger.info(MAX_BUCKETS_CREATED_MSG)
           end
 
-          def update_alerts(applications)
-            current_timestamp = Time.now.getutc
-
-            applications.each do |_appid, values|
-              service_id = values[:service_id]
-              application = Backend::Application.load(service_id,
-                                                      values[:application_id])
-
-              # The app could have been deleted at some point since the job was
-              # enqueued. No need to update alerts in that case.
-              next unless application
-
-              # The operations below are costly. They load all the usage limits
-              # and current usages to find the current utilization levels.
-              # That's why before that, we check if there are any alerts that
-              # can be raised.
-              next unless Alerts.can_raise_more_alerts?(service_id, values[:application_id])
-
-              application.load_metric_names
-              usage = Usage.application_usage(application, current_timestamp)
-              status = Transactor::Status.new(service_id: service_id,
-                                              application: application,
-                                              values: usage)
-
-              max_utilization, max_record = Alerts.utilization(
-                  status.application_usage_reports)
-
-              if max_utilization >= 0.0
-                Alerts.update_utilization(service_id,
-                                          values[:application_id],
-                                          max_utilization,
-                                          max_record,
-                                          current_timestamp)
+          def update_alerts(transactions)
+            transactions.group_by { |tx| tx.application_id }.each do |app_id, txs|
+              service_id = txs.first.service_id # All the txs of an app belong to the same service
+
+              # Finding the max utilization can be costly because it involves
+              # loading usage limits and current usages. That's why before that,
+              # we check if there are any alerts that can be raised.
+              next unless Alerts.can_raise_more_alerts?(service_id, app_id)
+
+              begin
+                max_utilization = if Alerts::UsagesChecked.need_to_check_all?(service_id, app_id)
+                                    Utilization.max_in_all_metrics(service_id, app_id).tap do
+                                      Alerts::UsagesChecked.mark_all_checked(service_id, app_id)
+                                    end
+                                  else
+                                    # metrics_ids here includes the metrics
+                                    # explicitly reported plus their parents in
+                                    # the hierarchy.
+                                    metric_ids = txs.map { |tx| tx.usage.keys }.flatten.uniq
+                                    Utilization.max_in_metrics(service_id, app_id, metric_ids)
+                                  end
+              rescue ApplicationNotFound
+                # The app could have been deleted at some point since the job
+                # was enqueued. No need to update alerts in that case.
+                next
+              end
+
+              if max_utilization && max_utilization.ratio > 0
+                Alerts.update_utilization(service_id, app_id, max_utilization)
               end
             end
           end

data/lib/3scale/backend/stats/cleaner.rb

@@ -36,12 +36,6 @@ module ThreeScale
         KEY_SERVICES_TO_DELETE = 'set_with_services_marked_for_deletion'.freeze
         private_constant :KEY_SERVICES_TO_DELETE
 
-        SLEEP_BETWEEN_SCANS = 0.01 # In seconds
-        private_constant :SLEEP_BETWEEN_SCANS
-
-        SCAN_SLICE = 500
-        private_constant :SCAN_SLICE
-
         STATS_KEY_PREFIX = 'stats/'.freeze
         private_constant :STATS_KEY_PREFIX
 

data/lib/3scale/backend/transactor/usage_report.rb

@@ -3,7 +3,7 @@ module ThreeScale
     module Transactor
       class Status
         class UsageReport
-          attr_reader :type, :period
+          attr_reader :type, :usage_limit, :period
 
           def initialize(status, usage_limit)
             @status      = status

data/lib/3scale/backend/usage.rb

@@ -3,12 +3,19 @@ module ThreeScale
     class Usage
       class << self
         def application_usage(application, timestamp)
-          usage(application, timestamp) do |metric_id, instance_period|
+          usage(application.usage_limits, timestamp) do |metric_id, instance_period|
             Stats::Keys.application_usage_value_key(
                 application.service_id, application.id, metric_id, instance_period)
           end
         end
 
+        def application_usage_for_limits(application, timestamp, usage_limits)
+          usage(usage_limits, timestamp) do |metric_id, instance_period|
+            Stats::Keys.application_usage_value_key(
+              application.service_id, application.id, metric_id, instance_period)
+          end
+        end
+
         def is_set?(usage_str)
           usage_str && usage_str[0] == '#'.freeze
         end
@@ -25,7 +32,7 @@ module ThreeScale
 
         private
 
-        def usage(obj, timestamp)
+        def usage(usage_limits, timestamp)
           # The timestamp does not change, so we can generate all the
           # instantiated periods just once.
           # This is important. Without this, the code can generate many instance
@@ -33,7 +40,7 @@ module ThreeScale
           # time.
           instance_periods = Period::instance_periods_for_ts(timestamp)
 
-          pairs = metric_period_pairs obj.usage_limits
+          pairs = metric_period_pairs usage_limits
           return {} if pairs.empty?
 
           keys = pairs.map do |(metric_id, period)|

data/lib/3scale/backend/utilization.rb

@@ -0,0 +1,83 @@
+module ThreeScale
+  module Backend
+    class Utilization
+      include Comparable
+
+      attr_reader :metric_id, :period, :max_value, :current_value
+
+      def initialize(limit, current_value)
+        @metric_id = limit.metric_id
+        @period = limit.period
+        @max_value = limit.value
+        @current_value = current_value
+        @encoded = encoded(limit, current_value)
+      end
+
+      def ratio
+        return 0 if max_value == 0 # Disabled metric
+        current_value/max_value.to_f
+      end
+
+      # Returns in the format needed by the Alerts class.
+      def to_s
+        @encoded
+      end
+
+      def <=>(other)
+        # Consider "disabled" the lowest ones
+        if ratio == 0 && other.ratio == 0
+          return max_value <=> other.max_value
+        end
+
+        ratio <=> other.ratio
+      end
+
+      # Note: this can return nil
+      def self.max_in_all_metrics(service_id, app_id)
+        application = Backend::Application.load!(service_id, app_id)
+
+        usage = Usage.application_usage(application, Time.now.getutc)
+
+        status = Transactor::Status.new(service_id: service_id,
+                                        application: application,
+                                        values: usage)
+
+        # Preloads all the metric names to avoid fetching them one by one when
+        # generating the usage reports
+        application.load_metric_names
+
+        max = status.application_usage_reports.map do |usage_report|
+          Utilization.new(usage_report.usage_limit, usage_report.current_value)
+        end.max
+
+        # Avoid returning a utilization for disabled metrics
+        max && max.max_value > 0 ? max : nil
+      end
+
+      # Note: this can return nil
+      def self.max_in_metrics(service_id, app_id, metric_ids)
+        application = Backend::Application.load!(service_id, app_id)
+
+        limits = UsageLimit.load_for_affecting_metrics(
+          service_id, application.plan_id, metric_ids
+        )
+
+        usage = Usage.application_usage_for_limits(application, Time.now.getutc, limits)
+
+        max = limits.map do |limit|
+          Utilization.new(limit, usage[limit.period][limit.metric_id])
+        end.max
+
+        # Avoid returning a utilization for disabled metrics
+        max && max.max_value > 0 ? max : nil
+      end
+
+      private
+
+      def encoded(limit, current_value)
+        metric_name = Metric.load_name(limit.service_id, limit.metric_id)
+        "#{metric_name} per #{limit.period}: #{current_value}/#{limit.value}"
+      end
+    end
+  end
+end

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.4.0'
+    VERSION = '3.4.1'
   end
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.4.0</version>
+        <version>3.4.1</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.4.1
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-14 00:00:00.000000000 Z
+date: 2021-06-17 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -164,6 +164,7 @@ files:
 - lib/3scale/backend/usage_limit.rb
 - lib/3scale/backend/use_cases/provider_key_change_use_case.rb
 - lib/3scale/backend/util.rb
+- lib/3scale/backend/utilization.rb
 - lib/3scale/backend/validators.rb
 - lib/3scale/backend/validators/base.rb
 - lib/3scale/backend/validators/key.rb