apisonator 3.3.1 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a812d330bdcb770421ed926c478c8a8b48087ba33ad28ab80317a48db935f432
-  data.tar.gz: 5bc8b40c69fbf5a6a680b71692dc66433746b053c0dbf3dcfe6e3d4aacd656af
+  metadata.gz: 78b328c64420fa58f4480eb39b555856dd9a032ba78eed32f084767988cea59d
+  data.tar.gz: de454b45677bc1c133eeda34c8fb4ebfe56253c4b4de851f1d192652c5baf999
 SHA512:
-  metadata.gz: 807c4d8cfc932e600780f4ba97fa5cfa3afcaca04ad0e27395e92fca31407d96e360412a290566c78955e5cf12895a5f4a70e589474de045bf3f0f7cf21311af
-  data.tar.gz: 17bf0d7c783ee36b78a885a441846edeaf8906185ab183c91ffc7f0c7f7c13e45b4237c0c7adb1a228794b3f64d4a0729f6eb8360a59910021d65a0b99c1edc1
+  metadata.gz: bf7613f456c9810217adfbdd5f345b86aa59d9abdccd44d308d846e7f271849d484601900348a4dcd4a16437d9e0376b7927e3675374a9be7a2af723c2820626
+  data.tar.gz: deb0823a0648f55bf24c24da6727afa4e62d2d5d92eeeee5545b9f6c8af38b2105bc8bb8caf3f7b28c187e020080b9b9f0b43721a42dfece2d6963d698a23e9a

data/CHANGELOG.md

@@ -2,6 +2,74 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.4.1 - 2021-06-16
+
+### Changed
+
+- Introduced performance optimizations around alerts detection
+([#287](https://github.com/3scale/apisonator/pull/287)).
+
+## 3.4.0 - 2021-06-14
+
+### Added
+
+- New extension that list the keys of an application
+([#284](https://github.com/3scale/apisonator/pull/284)).
+
+### Changed
+
+- It is now possible to use OIDC in the auth and authrep endpoints
+([#280](https://github.com/3scale/apisonator/pull/280)).
+- Updated multi-json to 1.15.0
+([#278](https://github.com/3scale/apisonator/pull/278)).
+
+### Removed
+
+- Deleted unused service attributes related with deleted end-users functionality
+([#277](https://github.com/3scale/apisonator/pull/277)).
+
+## 3.3.3 - 2021-03-09
+
+### Changed
+
+- Check if alerts can be raised before calculating utilization (perf
+optimization) ([#275](https://github.com/3scale/apisonator/pull/275)).
+
+### Removed
+
+- Stop maintaining unused "current_max" key in Alerts
+([#272](https://github.com/3scale/apisonator/pull/272)).
+
+## 3.3.2 - 2021-02-23
+
+### Fixed
+
+- Fixed nil exception in `Aggregator.process`
+([#269](https://github.com/3scale/apisonator/pull/269)).
+
+### Changed
+
+- Updated to Ruby 2.7 in Docker images
+([#265](https://github.com/3scale/apisonator/pull/265)) and
+([#266](https://github.com/3scale/apisonator/pull/266)).
+- Updated pry to 0.14.0 and pry-doc to 1.1.0
+([#267](https://github.com/3scale/apisonator/pull/267)).
+- Updated Docker image to be based on RHEL UBI 8
+([#268](https://github.com/3scale/apisonator/pull/268)).
+
+### Removed
+
+- Removed redundant prometheus config params
+(`listener_prometheus_metrics.enabled` and `listener_prometheus_metrics.port`)
+([#270](https://github.com/3scale/apisonator/pull/270)).
+
+## 3.3.1.1 - 2021-02-12
+
+### Changed
+
+- Updated our Puma fork to v4.3.7
+([#261](https://github.com/3scale/apisonator/pull/261)).
+
 ## 3.3.1 - 2021-02-11
 
 ### Fixed

data/Gemfile.base

@@ -36,8 +36,8 @@ end
 group :development do
   gem 'sshkit'
   gem 'source2swagger', git: 'https://github.com/3scale/source2swagger', branch: 'backend'
-  gem 'pry',      '~> 0.11.3'
-  gem 'pry-doc',  '~> 0.11.1'
+  gem 'pry',      '~> 0.14'
+  gem 'pry-doc',  '~> 1.1'
   gem 'license_finder', '~> 5'
 end
 
@@ -46,7 +46,7 @@ group :development, :test do
 end
 
 # Default server by platform
-gem 'puma', git: 'https://github.com/3scale/puma', ref: 'b034371406690d3e6c2a9301c4a48bd721f3efc3'
+gem 'puma', git: 'https://github.com/3scale/puma', branch: '3scale-4.3.7'
 # gems required by the runner
 gem 'gli', '~> 2.16.1', require: nil
 # Workers

data/Gemfile.lock

@@ -1,9 +1,10 @@
 GIT
   remote: https://github.com/3scale/puma
-  revision: b034371406690d3e6c2a9301c4a48bd721f3efc3
-  ref: b034371406690d3e6c2a9301c4a48bd721f3efc3
+  revision: c0601d08695839b8ffd0f380e91c3b91c1e8b754
+  branch: 3scale-4.3.7
   specs:
-    puma (2.15.3)
+    puma (4.3.7)
+      nio4r (~> 2.0)
 
 GIT
   remote: https://github.com/3scale/redis-rb
@@ -35,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.3.1)
+    apisonator (3.4.1)
 
 GEM
   remote: https://rubygems.org/
@@ -95,7 +96,7 @@ GEM
     chronic (0.10.2)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
@@ -130,13 +131,13 @@ GEM
     localhost (1.1.6)
     mapping (1.1.1)
     metaclass (0.0.4)
-    method_source (0.9.0)
+    method_source (1.0.0)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
-    multi_json (1.13.1)
+    multi_json (1.15.0)
     mustache (1.0.5)
     mustermann (1.0.2)
     net-scp (1.2.1)
@@ -163,15 +164,15 @@ GEM
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
     protocol-redis (0.5.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    pry (0.14.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     pry-byebug (3.5.1)
       byebug (~> 9.1)
       pry (~> 0.10)
-    pry-doc (0.11.1)
-      pry (~> 0.9)
-      yard (~> 0.9)
+    pry-doc (1.1.0)
+      pry (~> 0.11)
+      yard (~> 0.9.11)
     rack (2.1.4)
     rack-protection (2.0.3)
       rack
@@ -259,7 +260,7 @@ GEM
       prometheus-client (~> 1.0)
       yabeda (~> 0.5)
     yajl-ruby (1.3.1)
-    yard (0.9.20)
+    yard (0.9.26)
 
 PLATFORMS
   ruby
@@ -283,9 +284,9 @@ DEPENDENCIES
   nokogiri (~> 1.10.8)
   pg (= 0.20.0)
   pkg-config (~> 1.1.7)
-  pry (~> 0.11.3)
+  pry (~> 0.14)
   pry-byebug (~> 3.5.1)
-  pry-doc (~> 0.11.1)
+  pry-doc (~> 1.1)
   puma!
   rack (~> 2.1.4)
   rack-test (= 0.8.2)

data/Gemfile.on_prem.lock

@@ -1,9 +1,10 @@
 GIT
   remote: https://github.com/3scale/puma
-  revision: b034371406690d3e6c2a9301c4a48bd721f3efc3
-  ref: b034371406690d3e6c2a9301c4a48bd721f3efc3
+  revision: c0601d08695839b8ffd0f380e91c3b91c1e8b754
+  branch: 3scale-4.3.7
   specs:
-    puma (2.15.3)
+    puma (4.3.7)
+      nio4r (~> 2.0)
 
 GIT
   remote: https://github.com/3scale/redis-rb
@@ -35,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.3.1)
+    apisonator (3.4.1)
 
 GEM
   remote: https://rubygems.org/
@@ -85,7 +86,7 @@ GEM
     byebug (9.1.0)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
@@ -119,13 +120,13 @@ GEM
     localhost (1.1.6)
     mapping (1.1.1)
     metaclass (0.0.4)
-    method_source (0.9.0)
+    method_source (1.0.0)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
-    multi_json (1.13.1)
+    multi_json (1.15.0)
     mustache (1.0.5)
     mustermann (1.0.2)
     net-scp (1.2.1)
@@ -151,15 +152,15 @@ GEM
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
     protocol-redis (0.5.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    pry (0.14.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     pry-byebug (3.5.1)
       byebug (~> 9.1)
       pry (~> 0.10)
-    pry-doc (0.11.1)
-      pry (~> 0.9)
-      yard (~> 0.9)
+    pry-doc (1.1.0)
+      pry (~> 0.11)
+      yard (~> 0.9.11)
     rack (2.1.4)
     rack-protection (2.0.3)
       rack
@@ -243,7 +244,7 @@ GEM
       prometheus-client (~> 1.0)
       yabeda (~> 0.5)
     yajl-ruby (1.3.1)
-    yard (0.9.20)
+    yard (0.9.26)
 
 PLATFORMS
   ruby
@@ -264,9 +265,9 @@ DEPENDENCIES
   mocha (~> 1.3)
   nokogiri (~> 1.10.8)
   pkg-config (~> 1.1.7)
-  pry (~> 0.11.3)
+  pry (~> 0.14)
   pry-byebug (~> 3.5.1)
-  pry-doc (~> 0.11.1)
+  pry-doc (~> 1.1)
   puma!
   rack (~> 2.1.4)
   rack-test (= 0.8.2)

data/lib/3scale/backend.rb

@@ -44,6 +44,7 @@ require '3scale/backend/queue_storage'
 require '3scale/backend/errors'
 require '3scale/backend/stats'
 require '3scale/backend/usage_limit'
+require '3scale/backend/utilization'
 require '3scale/backend/alerts'
 require '3scale/backend/event_storage'
 require '3scale/backend/worker'

data/lib/3scale/backend/alert_limit.rb

@@ -1,21 +1,19 @@
 module ThreeScale
   module Backend
     class AlertLimit
-      module KeyHelpers
-        def key(service_id)
-          "alerts/service_id:#{service_id}/allowed_set"
-        end
-      end
-
-      include KeyHelpers
-      extend KeyHelpers
+      include Alerts::KeyHelpers
+      extend Alerts::KeyHelpers
 
       include Storable
 
       attr_accessor :service_id, :value
 
       def save
-        storage.sadd(key(service_id), value.to_i) if valid?
+        if valid?
+          res = storage.sadd(key_allowed_set(service_id), value.to_i)
+          Alerts::UsagesChecked.invalidate_for_service(service_id)
+          res
+        end
       end
 
       def to_hash
@@ -26,7 +24,7 @@ module ThreeScale
       end
 
       def self.load_all(service_id)
-        values = storage.smembers(key(service_id))
+        values = storage.smembers(key_allowed_set(service_id))
         values.map do |value|
           new(service_id: service_id, value: value.to_i)
         end
@@ -38,7 +36,7 @@ module ThreeScale
       end
 
       def self.delete(service_id, value)
-        storage.srem(key(service_id), value.to_i) if valid_value?(value)
+        storage.srem(key_allowed_set(service_id), value.to_i) if valid_value?(value)
       end
 
       def self.valid_value?(value)

data/lib/3scale/backend/alerts.rb

@@ -6,11 +6,10 @@ module ThreeScale
 
         # The compacted hour in the params refers to the
         # TimeHacks.to_compact_s method.
-        def alert_keys(service_id, app_id, discrete_utilization, compacted_hour_start)
+        def alert_keys(service_id, app_id, discrete_utilization)
           {
             already_notified: key_already_notified(service_id, app_id, discrete_utilization),
             allowed: key_allowed_set(service_id),
-            current_max: key_current_max(service_id, app_id, compacted_hour_start),
             current_id: key_current_id
           }
         end
@@ -31,18 +30,19 @@ module ThreeScale
           "#{prefix}allowed_set"
         end
 
-        def key_current_max(service_id, app_id, compacted_hour_start)
-          prefix = key_prefix(service_id, app_id)
-          "#{prefix}#{compacted_hour_start}/current_max"
-        end
-
         def key_current_id
           'alerts/current_id'.freeze
         end
+
+        def key_usage_already_checked(service_id, app_id)
+          prefix = key_prefix(service_id, app_id)
+          "#{prefix}usage_already_checked"
+        end
       end
 
       extend self
       extend KeyHelpers
+      include Memoizer::Decorator
 
       ALERT_TTL       = 24*3600 # 1 day (only one message per day)
       ## zero must be here and sorted, yes or yes
@@ -50,67 +50,104 @@ module ThreeScale
       FIRST_ALERT_BIN = ALERT_BINS.first
       RALERT_BINS     = ALERT_BINS.reverse.freeze
 
-      def utilization(app_usage_reports)
-        max_utilization = -1.0
-        max_record = nil
-        max = proc do |item|
-          if item.max_value > 0
-            utilization = item.current_value / item.max_value.to_f
-
-            if utilization > max_utilization
-              max_record = item
-              max_utilization = utilization
-            end
-          end
+      # This class is useful to reduce the amount of information that we need to
+      # fetch from Redis to determine whether an alert should be raised.
+      # In summary, alerts are raised at the application level and we need to
+      # wait for 24h before raising a new one for the same level (ALERTS_BIN
+      # above).
+      #
+      # This class allows us to check all the usage limits once and then not
+      # check all of them again (just the ones in the report job) until:
+      # 1) A specific alert has expired (24h passed since it was triggered).
+      # 2) A new alert bin is enabled for the service.
+      class UsagesChecked
+        extend KeyHelpers
+        extend StorageHelpers
+        include Memoizer::Decorator
+
+        def self.need_to_check_all?(service_id, app_id)
+          !storage.exists(key_usage_already_checked(service_id, app_id))
+        end
+        memoize :need_to_check_all?
+
+        def self.mark_all_checked(service_id, app_id)
+          ttl = ALERT_BINS.map do |bin|
+            ttl = storage.ttl(key_already_notified(service_id, app_id, bin))
+
+            # Redis returns -2 if key does not exist, and -1 if it exists without
+            # a TTL (we know this should not happen for the alert bins).
+            # In those cases we should just set the TTL to the max (ALERT_TTL).
+            ttl >= 0 ? ttl : ALERT_TTL
+          end.min
+
+          storage.setex(key_usage_already_checked(service_id, app_id), ttl, '1'.freeze)
+          Memoizer.clear(Memoizer.build_key(self, :need_to_check_all?, service_id, app_id))
+        end
+
+        def self.invalidate(service_id, app_id)
+          storage.del(key_usage_already_checked(service_id, app_id))
         end
 
-        app_usage_reports.each(&max)
+        def self.invalidate_for_service(service_id)
+          app_ids = []
+          cursor = 0
 
-        if max_utilization == -1
-          ## case that all the limits have max_value==0
-          max_utilization = 0
-          max_record = app_usage_reports.first
+          loop do
+            cursor, ids = storage.sscan(
+              Application.applications_set_key(service_id), cursor, count: SCAN_SLICE
+            )
+
+            app_ids += ids
+
+            break if cursor.to_i == 0
+          end
+
+          invalidate_batch(service_id, app_ids)
         end
 
-        [max_utilization, max_record]
+        def self.invalidate_batch(service_id, app_ids)
+          app_ids.each_slice(PIPELINED_SLICE_SIZE) do |ids|
+            keys = ids.map { |app_id| key_usage_already_checked(service_id, app_id) }
+            storage.del(keys)
+          end
+        end
+        private_class_method :invalidate_batch
       end
 
-      def update_utilization(service_id, app_id, max_utilization, max_record, timestamp)
-        discrete = utilization_discrete(max_utilization)
-        max_utilization_i = (max_utilization * 100.0).round
+      def can_raise_more_alerts?(service_id, app_id)
+        allowed_bins = allowed_set_for_service(service_id).sort
 
-        beginning_of_day = Period::Boundary.day_start(timestamp)
-        period_hour = Period::Boundary.hour_start(timestamp).to_compact_s
-        # UNIX timestamp for key expiration - add 1 day + 5 mins
-        expire_at = (beginning_of_day + 86700).to_i
+        return false if allowed_bins.empty?
 
-        keys = alert_keys(service_id, app_id, discrete, period_hour)
+        # If the bin with the highest value has already been notified, there's
+        # no need to notify anything else.
+        not notified?(service_id, app_id, allowed_bins.last)
+      end
 
-        already_alerted, allowed, current_max, _ = storage.pipelined do
+      def update_utilization(service_id, app_id, utilization)
+        discrete = utilization_discrete(utilization.ratio)
+
+        keys = alert_keys(service_id, app_id, discrete)
+
+        already_alerted, allowed = storage.pipelined do
           storage.get(keys[:already_notified])
           storage.sismember(keys[:allowed], discrete)
-          storage.get(keys[:current_max])
-          storage.expireat(keys[:current_max], expire_at)
-        end
-
-        ## update the status of utilization
-        if max_utilization_i > current_max.to_i
-          storage.set(keys[:current_max], max_utilization_i)
         end
 
         if already_alerted.nil? && allowed && discrete.to_i > 0
-          next_id, _ = storage.pipelined do
+          next_id, _, _ = storage.pipelined do
             storage.incr(keys[:current_id])
             storage.setex(keys[:already_notified], ALERT_TTL, "1")
+            UsagesChecked.invalidate(service_id, app_id)
           end
 
           alert = { :id => next_id,
                     :utilization => discrete,
-                    :max_utilization => max_utilization,
+                    :max_utilization => utilization.ratio,
                     :application_id => app_id,
                     :service_id => service_id,
-                    :timestamp => timestamp,
-                    :limit => formatted_limit(max_record) }
+                    :timestamp => Time.now.utc,
+                    :limit => utilization.to_s }
 
           Backend::EventStorage::store(:alert, alert)
         end
@@ -124,10 +161,15 @@ module ThreeScale
         end || FIRST_ALERT_BIN
       end
 
-      def formatted_limit(record)
-        "#{record.metric_name} per #{record.period}: "\
-        "#{record.current_value}/#{record.max_value}"
+      def allowed_set_for_service(service_id)
+        storage.smembers(key_allowed_set(service_id)).map(&:to_i) # Redis returns strings always
+      end
+      memoize :allowed_set_for_service
+
+      def notified?(service_id, app_id, bin)
+        storage.get(key_already_notified(service_id, app_id, bin))
       end
+      memoize :notified?
 
       def storage
         Storage.instance

data/lib/3scale/backend/configuration.rb

@@ -58,7 +58,6 @@ module ThreeScale
       config.add_section(:internal_api, :user, :password)
       config.add_section(:master, :metrics)
       config.add_section(:worker_prometheus_metrics, :enabled, :port)
-      config.add_section(:listener_prometheus_metrics, :enabled, :port)
 
       config.add_section(
           :async_worker,

data/lib/3scale/backend/constants.rb

@@ -4,6 +4,8 @@ module ThreeScale
       module All
         TIME_FORMAT          = '%Y-%m-%d %H:%M:%S %z'.freeze
         PIPELINED_SLICE_SIZE = 400
+        SLEEP_BETWEEN_SCANS = 0.01 # In seconds
+        SCAN_SLICE = 500
       end
 
       def self.included(base)

data/lib/3scale/backend/rack.rb

@@ -17,7 +17,10 @@ module ThreeScale
 
           Backend::Logging::External.setup_rack self
 
-          if Backend.configuration.listener_prometheus_metrics.enabled
+          # Notice that this cannot be specified via config, it needs to be an
+          # ENV because the metric server is started in Puma/Falcon
+          # "before_fork" and the configuration is not loaded at that point.
+          if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s.downcase.freeze == 'true'.freeze
             use Rack::Prometheus
           end
 

data/lib/3scale/backend/service.rb

@@ -4,16 +4,12 @@ module ThreeScale
       include Storable
 
       # list of attributes to be fetched from storage
-      ATTRIBUTES = %i[state referrer_filters_required backend_version
-                      user_registration_required default_user_plan_id
-                      default_user_plan_name provider_key].freeze
+      ATTRIBUTES = %i[state referrer_filters_required backend_version provider_key].freeze
       private_constant :ATTRIBUTES
 
       attr_reader :state
-      attr_accessor :provider_key, :id, :backend_version,
-        :default_user_plan_id, :default_user_plan_name
-      attr_writer :referrer_filters_required, :user_registration_required,
-        :default_service
+      attr_accessor :provider_key, :id, :backend_version
+      attr_writer :referrer_filters_required, :default_service
 
       class << self
         include Memoizer::Decorator
@@ -104,8 +100,6 @@ module ThreeScale
         memoize :list
 
         def save!(attributes = {})
-          massage_set_user_registration_required attributes
-
           new(attributes).save!
         end
 
@@ -139,26 +133,10 @@ module ThreeScale
         def massage_service_attrs(service_attrs)
           service_attrs[:referrer_filters_required] =
             service_attrs[:referrer_filters_required].to_i > 0
-          service_attrs[:user_registration_required] =
-            massage_get_user_registration_required(
-              service_attrs[:user_registration_required])
 
           service_attrs
         end
 
-        # nil => true, 1 => true, '1' => true, 0 => false, '0' => false
-        def massage_get_user_registration_required(value)
-          value.nil? ? true : value.to_i > 0
-        end
-
-        def massage_set_user_registration_required(attributes)
-          if attributes[:user_registration_required].nil?
-            val = storage.get(storage_key(attributes[:id], :user_registration_required))
-            attributes[:user_registration_required] =
-              (!val.nil? && val.to_i == 0) ? false : true
-          end
-        end
-
         def get_attr(id, attribute)
           storage.get(storage_key(id, attribute))
         end
@@ -195,10 +173,6 @@ module ThreeScale
         @referrer_filters_required
       end
 
-      def user_registration_required?
-        @user_registration_required
-      end
-
       def save!
         set_as_default_if_needed
         persist
@@ -227,9 +201,6 @@ module ThreeScale
           provider_key: provider_key,
           backend_version: backend_version,
           referrer_filters_required: referrer_filters_required?,
-          user_registration_required: user_registration_required?,
-          default_user_plan_id: default_user_plan_id,
-          default_user_plan_name: default_user_plan_name,
           default_service: default_service?
         }
       end
@@ -294,9 +265,6 @@ module ThreeScale
 
       def persist_attributes
         persist_attribute :referrer_filters_required, referrer_filters_required? ? 1 : 0
-        persist_attribute :user_registration_required, user_registration_required? ? 1 : 0
-        persist_attribute :default_user_plan_id, default_user_plan_id, true
-        persist_attribute :default_user_plan_name, default_user_plan_name, true
         persist_attribute :backend_version, backend_version, true
         persist_attribute :provider_key, provider_key
         persist_attribute :state, state.to_s if state

data/lib/3scale/backend/stats/aggregator.rb

@@ -59,7 +59,7 @@ module ThreeScale
             touched_apps = aggregate(transactions, current_bucket)
 
             ApplicationEvents.generate(touched_apps.values)
-            update_alerts(touched_apps)
+            update_alerts(transactions)
             begin
               ApplicationEvents.ping
             rescue ApplicationEvents::PingFailed => e
@@ -137,29 +137,35 @@ module ThreeScale
             logger.info(MAX_BUCKETS_CREATED_MSG)
           end
 
-          def update_alerts(applications)
-            current_timestamp = Time.now.getutc
-
-            applications.each do |_appid, values|
-              service_id = values[:service_id]
-              application = Backend::Application.load(service_id,
-                                                      values[:application_id])
-
-              application.load_metric_names
-              usage = Usage.application_usage(application, current_timestamp)
-              status = Transactor::Status.new(service_id: service_id,
-                                              application: application,
-                                              values: usage)
-
-              max_utilization, max_record = Alerts.utilization(
-                  status.application_usage_reports)
-
-              if max_utilization >= 0.0
-                Alerts.update_utilization(service_id,
-                                          values[:application_id],
-                                          max_utilization,
-                                          max_record,
-                                          current_timestamp)
+          def update_alerts(transactions)
+            transactions.group_by { |tx| tx.application_id }.each do |app_id, txs|
+              service_id = txs.first.service_id # All the txs of an app belong to the same service
+
+              # Finding the max utilization can be costly because it involves
+              # loading usage limits and current usages. That's why before that,
+              # we check if there are any alerts that can be raised.
+              next unless Alerts.can_raise_more_alerts?(service_id, app_id)
+
+              begin
+                max_utilization = if Alerts::UsagesChecked.need_to_check_all?(service_id, app_id)
+                                    Utilization.max_in_all_metrics(service_id, app_id).tap do
+                                      Alerts::UsagesChecked.mark_all_checked(service_id, app_id)
+                                    end
+                                  else
+                                    # metrics_ids here includes the metrics
+                                    # explicitly reported plus their parents in
+                                    # the hierarchy.
+                                    metric_ids = txs.map { |tx| tx.usage.keys }.flatten.uniq
+                                    Utilization.max_in_metrics(service_id, app_id, metric_ids)
+                                  end
+              rescue ApplicationNotFound
+                # The app could have been deleted at some point since the job
+                # was enqueued. No need to update alerts in that case.
+                next
+              end
+
+              if max_utilization && max_utilization.ratio > 0
+                Alerts.update_utilization(service_id, app_id, max_utilization)
               end
             end
           end

data/lib/3scale/backend/stats/cleaner.rb

@@ -36,12 +36,6 @@ module ThreeScale
         KEY_SERVICES_TO_DELETE = 'set_with_services_marked_for_deletion'.freeze
         private_constant :KEY_SERVICES_TO_DELETE
 
-        SLEEP_BETWEEN_SCANS = 0.01 # In seconds
-        private_constant :SLEEP_BETWEEN_SCANS
-
-        SCAN_SLICE = 500
-        private_constant :SCAN_SLICE
-
         STATS_KEY_PREFIX = 'stats/'.freeze
         private_constant :STATS_KEY_PREFIX
 

data/lib/3scale/backend/transactor.rb

@@ -61,6 +61,8 @@ module ThreeScale
 
       def validate(oauth, provider_key, report_usage, params, request_info)
         service = Service.load_with_provider_key!(params[:service_id], provider_key)
+        oidc_service = !oauth && service.backend_version == 'oauth'.freeze
+
         # service_id cannot be taken from params since it might be missing there
         service_id = service.id
 
@@ -70,12 +72,18 @@ module ThreeScale
         # significant.
         params[:app_id] = nil if app_id && app_id.empty?
 
-        if oauth
-          raise ApplicationNotFound.new nil if app_id.nil?
-          validators = Validators::OAUTH_VALIDATORS
-        else
-          validators = Validators::VALIDATORS
-        end
+        # While OIDC without an app_id makes little sense, we would break existing
+        # behaviour when calling non oauth_auth*.xml endpoints if we returned an
+        # error here, so only do this for oauth_auth*.xml endpoints.
+        raise ApplicationNotFound.new nil if oauth && app_id.nil?
+
+        validators = if oidc_service
+                       Validators::OIDC_VALIDATORS
+                     elsif oauth
+                       Validators::OAUTH_VALIDATORS
+                     else
+                       Validators::VALIDATORS
+                     end
 
         params[:user_key] = nil if params[:user_key] && params[:user_key].empty?
         application = Application.load_by_id_or_user_key!(service_id,
@@ -98,8 +106,9 @@ module ThreeScale
           # hierarchy parameter adds information in the response needed
           # to derive which limits affect directly or indirectly the
           # metrics for which authorization is requested.
-          hierarchy:       extensions[:hierarchy] == '1',
-          flat_usage:      extensions[:flat_usage] == '1'
+          hierarchy:       extensions[:hierarchy] == '1'.freeze,
+          flat_usage:      extensions[:flat_usage] == '1'.freeze,
+          list_app_keys:   extensions[:list_app_keys] == '1'.freeze
         }
 
         application.load_metric_names
@@ -108,24 +117,6 @@ module ThreeScale
         apply_validators(validators, status_attrs, params)
       end
 
-      def get_token_ids(token, service_id, app_id)
-        begin
-          token_aid = OAuth::Token::Storage.get_credentials(token, service_id)
-        rescue AccessTokenInvalid => e
-          # Yep, well, er. Someone specified that it is OK to have an
-          # invalid token if an app_id is specified. Somehow passing in
-          # a user_key is still not enough, though...
-          raise e if app_id.nil?
-        end
-
-        # We only take the token ids into account if we had no parameter ids
-        if app_id.nil?
-          app_id = token_aid
-        end
-
-        app_id
-      end
-
       def do_authorize(method, provider_key, params, context_info)
         notify_authorize(provider_key)
         validate(method == :oauth_authorize, provider_key, false, params, context_info[:request])

data/lib/3scale/backend/transactor/status.rb

@@ -8,17 +8,23 @@ module ThreeScale
         # We only use 'redirect_uri' if a request sent such a param. See #397.
         REDIRECT_URI_FIELD = 'redirect_url'.freeze
         private_constant :REDIRECT_URI_FIELD
+        # Maximum number of keys to list when using the list_app_keys extension
+        # At the time of writing System/Porta has a limit of 5 different app_keys
+        # at any given moment, but this could change anytime.
+        LIST_APP_KEYS_MAX = 256
+        private_constant :LIST_APP_KEYS_MAX
 
         def initialize(attributes)
-          @service_id      = attributes[:service_id]
-          @application     = attributes[:application]
-          @oauth           = attributes[:oauth]
-          @usage           = attributes[:usage]
-          @predicted_usage = attributes[:predicted_usage]
-          @values          = filter_values(attributes[:values] || {})
-          @timestamp       = attributes[:timestamp] || Time.now.getutc
-          @hierarchy_ext   = attributes[:hierarchy]
-          @flat_usage_ext  = attributes[:flat_usage]
+          @service_id        = attributes[:service_id]
+          @application       = attributes[:application]
+          @oauth             = attributes[:oauth]
+          @usage             = attributes[:usage]
+          @predicted_usage   = attributes[:predicted_usage]
+          @values            = filter_values(attributes[:values] || {})
+          @timestamp         = attributes[:timestamp] || Time.now.getutc
+          @hierarchy_ext     = attributes[:hierarchy]
+          @flat_usage_ext    = attributes[:flat_usage]
+          @list_app_keys_ext = attributes[:list_app_keys]
 
           raise 'service_id not specified' if @service_id.nil?
           raise ':application is required' if @application.nil?
@@ -106,6 +112,7 @@ module ThreeScale
             add_plan_section(xml, 'plan'.freeze, plan_name)
             add_reports_section(xml, application_usage_reports)
             hierarchy_reports.concat application_usage_reports if hierarchy_reports
+            add_app_keys_section xml if @list_app_keys_ext
           end
 
           if hierarchy_reports
@@ -161,6 +168,17 @@ module ThreeScale
           xml << '</hierarchy>'.freeze
         end
 
+        def add_app_keys_section(xml)
+          xml << '<app_keys app="'.freeze
+          xml << @application.id << '" svc="'.freeze
+          xml << @service_id << '">'.freeze
+          @application.keys.take(LIST_APP_KEYS_MAX).each do |key|
+            xml << '<key id="'.freeze
+            xml << key << '"/>'.freeze
+          end
+          xml << '</app_keys>'.freeze
+        end
+
         # helper to iterate over reports and get relevant hierarchy info
         def with_report_and_hierarchy(reports)
           reports.each do |ur|

data/lib/3scale/backend/transactor/usage_report.rb

@@ -3,7 +3,7 @@ module ThreeScale
     module Transactor
       class Status
         class UsageReport
-          attr_reader :type, :period
+          attr_reader :type, :usage_limit, :period
 
           def initialize(status, usage_limit)
             @status      = status

data/lib/3scale/backend/usage.rb

@@ -3,12 +3,19 @@ module ThreeScale
     class Usage
       class << self
         def application_usage(application, timestamp)
-          usage(application, timestamp) do |metric_id, instance_period|
+          usage(application.usage_limits, timestamp) do |metric_id, instance_period|
             Stats::Keys.application_usage_value_key(
                 application.service_id, application.id, metric_id, instance_period)
           end
         end
 
+        def application_usage_for_limits(application, timestamp, usage_limits)
+          usage(usage_limits, timestamp) do |metric_id, instance_period|
+            Stats::Keys.application_usage_value_key(
+              application.service_id, application.id, metric_id, instance_period)
+          end
+        end
+
         def is_set?(usage_str)
           usage_str && usage_str[0] == '#'.freeze
         end
@@ -25,7 +32,7 @@ module ThreeScale
 
         private
 
-        def usage(obj, timestamp)
+        def usage(usage_limits, timestamp)
           # The timestamp does not change, so we can generate all the
           # instantiated periods just once.
           # This is important. Without this, the code can generate many instance
@@ -33,7 +40,7 @@ module ThreeScale
           # time.
           instance_periods = Period::instance_periods_for_ts(timestamp)
 
-          pairs = metric_period_pairs obj.usage_limits
+          pairs = metric_period_pairs usage_limits
           return {} if pairs.empty?
 
           keys = pairs.map do |(metric_id, period)|

data/lib/3scale/backend/utilization.rb

@@ -0,0 +1,83 @@
+module ThreeScale
+  module Backend
+    class Utilization
+      include Comparable
+
+      attr_reader :metric_id, :period, :max_value, :current_value
+
+      def initialize(limit, current_value)
+        @metric_id = limit.metric_id
+        @period = limit.period
+        @max_value = limit.value
+        @current_value = current_value
+        @encoded = encoded(limit, current_value)
+      end
+
+      def ratio
+        return 0 if max_value == 0 # Disabled metric
+        current_value/max_value.to_f
+      end
+
+      # Returns in the format needed by the Alerts class.
+      def to_s
+        @encoded
+      end
+
+      def <=>(other)
+        # Consider "disabled" the lowest ones
+        if ratio == 0 && other.ratio == 0
+          return max_value <=> other.max_value
+        end
+
+        ratio <=> other.ratio
+      end
+
+      # Note: this can return nil
+      def self.max_in_all_metrics(service_id, app_id)
+        application = Backend::Application.load!(service_id, app_id)
+
+        usage = Usage.application_usage(application, Time.now.getutc)
+
+        status = Transactor::Status.new(service_id: service_id,
+                                        application: application,
+                                        values: usage)
+
+        # Preloads all the metric names to avoid fetching them one by one when
+        # generating the usage reports
+        application.load_metric_names
+
+        max = status.application_usage_reports.map do |usage_report|
+          Utilization.new(usage_report.usage_limit, usage_report.current_value)
+        end.max
+
+        # Avoid returning a utilization for disabled metrics
+        max && max.max_value > 0 ? max : nil
+      end
+
+      # Note: this can return nil
+      def self.max_in_metrics(service_id, app_id, metric_ids)
+        application = Backend::Application.load!(service_id, app_id)
+
+        limits = UsageLimit.load_for_affecting_metrics(
+          service_id, application.plan_id, metric_ids
+        )
+
+        usage = Usage.application_usage_for_limits(application, Time.now.getutc, limits)
+
+        max = limits.map do |limit|
+          Utilization.new(limit, usage[limit.period][limit.metric_id])
+        end.max
+
+        # Avoid returning a utilization for disabled metrics
+        max && max.max_value > 0 ? max : nil
+      end
+
+      private
+
+      def encoded(limit, current_value)
+        metric_name = Metric.load_name(limit.service_id, limit.metric_id)
+        "#{metric_name} per #{limit.period}: #{current_value}/#{limit.value}"
+      end
+    end
+  end
+end

data/lib/3scale/backend/validators.rb

@@ -21,6 +21,13 @@ module ThreeScale
       OAUTH_VALIDATORS = ([Validators::OauthSetting,
                            Validators::OauthKey,
                            Validators::RedirectURI] + COMMON_VALIDATORS).freeze
+
+      # OIDC specific validators will only check app keys when app_key is given.
+      #
+      # No need to add OauthSetting, since we need to check that to tell
+      # OIDC apart from the rest when calling authrep.xml (note lack of
+      # the oauth_ prefix).
+      OIDC_VALIDATORS = ([Validators::OauthKey] + COMMON_VALIDATORS).freeze
     end
   end
 end

data/lib/3scale/backend/validators/oauth_setting.rb

@@ -3,7 +3,7 @@ module ThreeScale
     module Validators
       class OauthSetting < Base
         def apply
-          if service.backend_version == 'oauth'
+          if service.backend_version == 'oauth'.freeze
             succeed!
           else
             fail!(OauthNotEnabled.new)

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.3.1'
+    VERSION = '3.4.1'
   end
 end

data/lib/3scale/prometheus_server.rb

@@ -4,7 +4,7 @@
 require_relative '../3scale/backend/listener_metrics'
 
 # Config is not loaded at this point, so read ENV instead.
-if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s == 'true'
+if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s.downcase.freeze == 'true'.freeze
   prometheus_port = ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_PORT']
   ThreeScale::Backend::ListenerMetrics.start_metrics_server(prometheus_port)
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.3.1</version>
+        <version>3.4.1</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -233,7 +233,7 @@
       </dependency>
           <dependency>
         <packageName>coderay</packageName>
-        <version>1.1.2</version>
+        <version>1.1.3</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -421,7 +421,7 @@
       </dependency>
           <dependency>
         <packageName>method_source</packageName>
-        <version>0.9.0</version>
+        <version>1.0.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -475,7 +475,7 @@
       </dependency>
           <dependency>
         <packageName>multi_json</packageName>
-        <version>1.13.1</version>
+        <version>1.15.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -679,7 +679,7 @@
       </dependency>
           <dependency>
         <packageName>pry</packageName>
-        <version>0.11.3</version>
+        <version>0.14.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -699,7 +699,7 @@
       </dependency>
           <dependency>
         <packageName>pry-doc</packageName>
-        <version>0.11.1</version>
+        <version>1.1.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -709,7 +709,7 @@
       </dependency>
           <dependency>
         <packageName>puma</packageName>
-        <version>2.15.3</version>
+        <version>4.3.7</version>
         <licenses>
                       <license>
               <name>New BSD</name>
@@ -1143,7 +1143,7 @@
       </dependency>
           <dependency>
         <packageName>yard</packageName>
-        <version>0.9.20</version>
+        <version>0.9.26</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.4.1
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-06-17 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -164,6 +164,7 @@ files:
 - lib/3scale/backend/usage_limit.rb
 - lib/3scale/backend/use_cases/provider_key_change_use_case.rb
 - lib/3scale/backend/util.rb
+- lib/3scale/backend/utilization.rb
 - lib/3scale/backend/validators.rb
 - lib/3scale/backend/validators/base.rb
 - lib/3scale/backend/validators/key.rb
@@ -207,8 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.7
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.2.10
 signing_key: 
 specification_version: 4
 summary: 3scale web service management system backend