apisonator 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8e132216edb5cd3b44967acab917d4ca4f50c692c22fb5b81c7ffff9c5d598b
-  data.tar.gz: e2e31c92b450398dbcd3a3dcc3d31ed39b2d530048de23aa06b95c4930809249
+  metadata.gz: 7a260bc57e82268c54885b58a12c79d28c9b7c53f8fb5e48182c887488888184
+  data.tar.gz: 7dbed84c520c06e914d15e99f6e1b78365a1b695d8cf8e1fd66a58c02fa36fa9
 SHA512:
-  metadata.gz: 288b9c506a32caf81e2b3bcbf06b347e1f39f32a295a61d146ebc9b7f5953458f27114c3eceb08ef00da6293487d02f49b5c7688af16f87c0aa0ff4c70cfa48f
-  data.tar.gz: 025f8fcf1edb087de5e97ad43ecca61e25c97f061f0002b75d210a1052fa76355c5ea8a34515018d41b21428791d255341e2f36d2c178872ae549d3f97f2e52e
+  metadata.gz: 4d0bdd90d90972bc416faf04a852ff56c3895da1cb191b264617f71add2f22c5942d928d85c9877faceab4f00e7fe26955dfeed7c6cbc615a8fe555b7c6e251a
+  data.tar.gz: 22b6dbdb74d73117ba7358d231b5f747f54746687e452285b77883df4b352d20d59b3997b38a0c5c7b6cba9bce618b41900d6a0ee3ea1e169727470bce8158cb

data/CHANGELOG.md

@@ -2,6 +2,74 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.4.0 - 2021-06-14
+
+### Added
+
+- New extension that list the keys of an application
+([#284](https://github.com/3scale/apisonator/pull/284)).
+
+### Changed
+
+- It is now possible to use OIDC in the auth and authrep endpoints
+([#280](https://github.com/3scale/apisonator/pull/280)).
+- Updated multi-json to 1.15.0
+([#278](https://github.com/3scale/apisonator/pull/278)).
+
+### Removed
+
+- Deleted unused service attributes related with deleted end-users functionality
+([#277](https://github.com/3scale/apisonator/pull/277)).
+
+## 3.3.3 - 2021-03-09
+
+### Changed
+
+- Check if alerts can be raised before calculating utilization (perf
+optimization) ([#275](https://github.com/3scale/apisonator/pull/275)).
+
+### Removed
+
+- Stop maintaining unused "current_max" key in Alerts
+([#272](https://github.com/3scale/apisonator/pull/272)).
+
+## 3.3.2 - 2021-02-23
+
+### Fixed
+
+- Fixed nil exception in `Aggregator.process`
+([#269](https://github.com/3scale/apisonator/pull/269)).
+
+### Changed
+
+- Updated to Ruby 2.7 in Docker images
+([#265](https://github.com/3scale/apisonator/pull/265)) and
+([#266](https://github.com/3scale/apisonator/pull/266)).
+- Updated pry to 0.14.0 and pry-doc to 1.1.0
+([#267](https://github.com/3scale/apisonator/pull/267)).
+- Updated Docker image to be based on RHEL UBI 8
+([#268](https://github.com/3scale/apisonator/pull/268)).
+
+### Removed
+
+- Removed redundant prometheus config params
+(`listener_prometheus_metrics.enabled` and `listener_prometheus_metrics.port`)
+([#270](https://github.com/3scale/apisonator/pull/270)).
+
+## 3.3.1.1 - 2021-02-12
+
+### Changed
+
+- Updated our Puma fork to v4.3.7
+([#261](https://github.com/3scale/apisonator/pull/261)).
+
+## 3.3.1 - 2021-02-11
+
+### Fixed
+
+- Usages with `#0` (set to 0) no longer generate unnecessary stats keys in Redis
+([#258](https://github.com/3scale/apisonator/pull/258)).
+
 ## 3.3.0 - 2021-02-09
 
 ### Added

data/Gemfile.base

@@ -15,11 +15,15 @@ platform :ruby do
 end
 
 group :test do
+  # Newer versions of rack-test don't work well with rspec-api-documentation.
+  # See https://github.com/rack/rack-test/pull/223 &
+  # https://github.com/zipmark/rspec_api_documentation/issues/342
+  gem 'rack-test',     '= 0.8.2'
+
   gem 'benchmark-ips', '~> 2.7.2'
   gem 'mocha',         '~> 1.3'
   gem 'nokogiri',      '~> 1.10.8'
   gem 'pkg-config',    '~> 1.1.7'
-  gem 'rack-test',     '~> 0.8.2'
   gem 'resque_unit',   '~> 0.4.4', source: 'https://rubygems.org'
   gem 'test-unit',     '~> 3.2.6'
   gem 'resque_spec',   '~> 0.17.0'
@@ -32,8 +36,8 @@ end
 group :development do
   gem 'sshkit'
   gem 'source2swagger', git: 'https://github.com/3scale/source2swagger', branch: 'backend'
-  gem 'pry',      '~> 0.11.3'
-  gem 'pry-doc',  '~> 0.11.1'
+  gem 'pry',      '~> 0.14'
+  gem 'pry-doc',  '~> 1.1'
   gem 'license_finder', '~> 5'
 end
 
@@ -42,7 +46,7 @@ group :development, :test do
 end
 
 # Default server by platform
-gem 'puma', git: 'https://github.com/3scale/puma', ref: 'b034371406690d3e6c2a9301c4a48bd721f3efc3'
+gem 'puma', git: 'https://github.com/3scale/puma', branch: '3scale-4.3.7'
 # gems required by the runner
 gem 'gli', '~> 2.16.1', require: nil
 # Workers

data/Gemfile.lock

@@ -1,9 +1,10 @@
 GIT
   remote: https://github.com/3scale/puma
-  revision: b034371406690d3e6c2a9301c4a48bd721f3efc3
-  ref: b034371406690d3e6c2a9301c4a48bd721f3efc3
+  revision: c0601d08695839b8ffd0f380e91c3b91c1e8b754
+  branch: 3scale-4.3.7
   specs:
-    puma (2.15.3)
+    puma (4.3.7)
+      nio4r (~> 2.0)
 
 GIT
   remote: https://github.com/3scale/redis-rb
@@ -35,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.3.0)
+    apisonator (3.4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -95,7 +96,7 @@ GEM
     chronic (0.10.2)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
@@ -130,13 +131,13 @@ GEM
     localhost (1.1.6)
     mapping (1.1.1)
     metaclass (0.0.4)
-    method_source (0.9.0)
+    method_source (1.0.0)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
-    multi_json (1.13.1)
+    multi_json (1.15.0)
     mustache (1.0.5)
     mustermann (1.0.2)
     net-scp (1.2.1)
@@ -163,15 +164,15 @@ GEM
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
     protocol-redis (0.5.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    pry (0.14.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     pry-byebug (3.5.1)
       byebug (~> 9.1)
       pry (~> 0.10)
-    pry-doc (0.11.1)
-      pry (~> 0.9)
-      yard (~> 0.9)
+    pry-doc (1.1.0)
+      pry (~> 0.11)
+      yard (~> 0.9.11)
     rack (2.1.4)
     rack-protection (2.0.3)
       rack
@@ -259,7 +260,7 @@ GEM
       prometheus-client (~> 1.0)
       yabeda (~> 0.5)
     yajl-ruby (1.3.1)
-    yard (0.9.20)
+    yard (0.9.26)
 
 PLATFORMS
   ruby
@@ -283,12 +284,12 @@ DEPENDENCIES
   nokogiri (~> 1.10.8)
   pg (= 0.20.0)
   pkg-config (~> 1.1.7)
-  pry (~> 0.11.3)
+  pry (~> 0.14)
   pry-byebug (~> 3.5.1)
-  pry-doc (~> 0.11.1)
+  pry-doc (~> 1.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
   redis-namespace (~> 1.8.0)

data/Gemfile.on_prem.lock

@@ -1,9 +1,10 @@
 GIT
   remote: https://github.com/3scale/puma
-  revision: b034371406690d3e6c2a9301c4a48bd721f3efc3
-  ref: b034371406690d3e6c2a9301c4a48bd721f3efc3
+  revision: c0601d08695839b8ffd0f380e91c3b91c1e8b754
+  branch: 3scale-4.3.7
   specs:
-    puma (2.15.3)
+    puma (4.3.7)
+      nio4r (~> 2.0)
 
 GIT
   remote: https://github.com/3scale/redis-rb
@@ -35,7 +36,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.3.0)
+    apisonator (3.4.0)
 
 GEM
   remote: https://rubygems.org/
@@ -85,7 +86,7 @@ GEM
     byebug (9.1.0)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
@@ -119,13 +120,13 @@ GEM
     localhost (1.1.6)
     mapping (1.1.1)
     metaclass (0.0.4)
-    method_source (0.9.0)
+    method_source (1.0.0)
     mini_portile2 (2.4.0)
     minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
-    multi_json (1.13.1)
+    multi_json (1.15.0)
     mustache (1.0.5)
     mustermann (1.0.2)
     net-scp (1.2.1)
@@ -151,15 +152,15 @@ GEM
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
     protocol-redis (0.5.0)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    pry (0.14.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     pry-byebug (3.5.1)
       byebug (~> 9.1)
       pry (~> 0.10)
-    pry-doc (0.11.1)
-      pry (~> 0.9)
-      yard (~> 0.9)
+    pry-doc (1.1.0)
+      pry (~> 0.11)
+      yard (~> 0.9.11)
     rack (2.1.4)
     rack-protection (2.0.3)
       rack
@@ -243,7 +244,7 @@ GEM
       prometheus-client (~> 1.0)
       yabeda (~> 0.5)
     yajl-ruby (1.3.1)
-    yard (0.9.20)
+    yard (0.9.26)
 
 PLATFORMS
   ruby
@@ -264,12 +265,12 @@ DEPENDENCIES
   mocha (~> 1.3)
   nokogiri (~> 1.10.8)
   pkg-config (~> 1.1.7)
-  pry (~> 0.11.3)
+  pry (~> 0.14)
   pry-byebug (~> 3.5.1)
-  pry-doc (~> 0.11.1)
+  pry-doc (~> 1.1)
   puma!
   rack (~> 2.1.4)
-  rack-test (~> 0.8.2)
+  rack-test (= 0.8.2)
   rake (~> 13.0)
   redis!
   redis-namespace (~> 1.8.0)

data/app/api/internal/stats.rb

@@ -6,32 +6,13 @@ module ThreeScale
           respond_with_404('service not found') unless Service.exists?(params[:service_id])
         end
 
-        # This is very slow and needs to be disabled until the performance
-        # issues are solved. In the meanwhile, the job will just return OK.
-=begin
-        delete '' do |service_id|
-          delete_stats_job_attrs = api_params Stats::DeleteJobDef
-          delete_stats_job_attrs[:service_id] = service_id
-          delete_stats_job_attrs[:from] = delete_stats_job_attrs[:from].to_i
-          delete_stats_job_attrs[:to] = delete_stats_job_attrs[:to].to_i
-          begin
-            Stats::DeleteJobDef.new(delete_stats_job_attrs).run_async
-          rescue DeleteServiceStatsValidationError => e
-            [400, headers, { status: :error, error: e.message }.to_json]
-          else
-            { status: :to_be_deleted }.to_json
-          end
-=end
-
-        # This is an alternative to the above. It just adds the service to a
-        # Redis set to marked is as "to be deleted".
-        # Later a script can read that set and actually delete the keys.
-        # Read the docs of the Stats::Cleaner class for more details.
+        # This adds the service to a Redis set to mark is as "to be deleted".
+        # Later a script can read that set and actually delete the keys. Read
+        # the docs of the Stats::Cleaner class for more details.
         #
-        # Notice that this method ignores the "from" and "to" parameters. When
-        # system calls this method, they're always interested in deleting all
-        # the keys. They were just passing "from" and "to" to make the
-        # implementation of the option above easier.
+        # Notice that this method ignores the "from" and "to" parameters used in
+        # previous versions. When system calls this method, they're always
+        # interested in deleting all the keys.
         delete '' do |service_id|
           Stats::Cleaner.mark_service_to_be_deleted(service_id)
           { status: :to_be_deleted }.to_json

data/lib/3scale/backend/alert_limit.rb

@@ -1,21 +1,15 @@
 module ThreeScale
   module Backend
     class AlertLimit
-      module KeyHelpers
-        def key(service_id)
-          "alerts/service_id:#{service_id}/allowed_set"
-        end
-      end
-
-      include KeyHelpers
-      extend KeyHelpers
+      include Alerts::KeyHelpers
+      extend Alerts::KeyHelpers
 
       include Storable
 
       attr_accessor :service_id, :value
 
       def save
-        storage.sadd(key(service_id), value.to_i) if valid?
+        storage.sadd(key_allowed_set(service_id), value.to_i) if valid?
       end
 
       def to_hash
@@ -26,7 +20,7 @@ module ThreeScale
       end
 
       def self.load_all(service_id)
-        values = storage.smembers(key(service_id))
+        values = storage.smembers(key_allowed_set(service_id))
         values.map do |value|
           new(service_id: service_id, value: value.to_i)
         end
@@ -38,7 +32,7 @@ module ThreeScale
       end
 
       def self.delete(service_id, value)
-        storage.srem(key(service_id), value.to_i) if valid_value?(value)
+        storage.srem(key_allowed_set(service_id), value.to_i) if valid_value?(value)
       end
 
       def self.valid_value?(value)

data/lib/3scale/backend/alerts.rb

@@ -6,11 +6,10 @@ module ThreeScale
 
         # The compacted hour in the params refers to the
         # TimeHacks.to_compact_s method.
-        def alert_keys(service_id, app_id, discrete_utilization, compacted_hour_start)
+        def alert_keys(service_id, app_id, discrete_utilization)
           {
             already_notified: key_already_notified(service_id, app_id, discrete_utilization),
             allowed: key_allowed_set(service_id),
-            current_max: key_current_max(service_id, app_id, compacted_hour_start),
             current_id: key_current_id
           }
         end
@@ -31,11 +30,6 @@ module ThreeScale
           "#{prefix}allowed_set"
         end
 
-        def key_current_max(service_id, app_id, compacted_hour_start)
-          prefix = key_prefix(service_id, app_id)
-          "#{prefix}#{compacted_hour_start}/current_max"
-        end
-
         def key_current_id
           'alerts/current_id'.freeze
         end
@@ -43,6 +37,7 @@ module ThreeScale
 
       extend self
       extend KeyHelpers
+      include Memoizer::Decorator
 
       ALERT_TTL       = 24*3600 # 1 day (only one message per day)
       ## zero must be here and sorted, yes or yes
@@ -50,6 +45,16 @@ module ThreeScale
       FIRST_ALERT_BIN = ALERT_BINS.first
       RALERT_BINS     = ALERT_BINS.reverse.freeze
 
+      def can_raise_more_alerts?(service_id, app_id)
+        allowed_bins = allowed_set_for_service(service_id).sort
+
+        return false if allowed_bins.empty?
+
+        # If the bin with the highest value has already been notified, there's
+        # no need to notify anything else.
+        not notified?(service_id, app_id, allowed_bins.last)
+      end
+
       def utilization(app_usage_reports)
         max_utilization = -1.0
         max_record = nil
@@ -77,25 +82,12 @@ module ThreeScale
 
       def update_utilization(service_id, app_id, max_utilization, max_record, timestamp)
         discrete = utilization_discrete(max_utilization)
-        max_utilization_i = (max_utilization * 100.0).round
 
-        beginning_of_day = Period::Boundary.day_start(timestamp)
-        period_hour = Period::Boundary.hour_start(timestamp).to_compact_s
-        # UNIX timestamp for key expiration - add 1 day + 5 mins
-        expire_at = (beginning_of_day + 86700).to_i
+        keys = alert_keys(service_id, app_id, discrete)
 
-        keys = alert_keys(service_id, app_id, discrete, period_hour)
-
-        already_alerted, allowed, current_max, _ = storage.pipelined do
+        already_alerted, allowed = storage.pipelined do
           storage.get(keys[:already_notified])
           storage.sismember(keys[:allowed], discrete)
-          storage.get(keys[:current_max])
-          storage.expireat(keys[:current_max], expire_at)
-        end
-
-        ## update the status of utilization
-        if max_utilization_i > current_max.to_i
-          storage.set(keys[:current_max], max_utilization_i)
         end
 
         if already_alerted.nil? && allowed && discrete.to_i > 0
@@ -129,6 +121,16 @@ module ThreeScale
         "#{record.current_value}/#{record.max_value}"
       end
 
+      def allowed_set_for_service(service_id)
+        storage.smembers(key_allowed_set(service_id)).map(&:to_i) # Redis returns strings always
+      end
+      memoize :allowed_set_for_service
+
+      def notified?(service_id, app_id, bin)
+        storage.get(key_already_notified(service_id, app_id, bin))
+      end
+      memoize :notified?
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/configuration.rb

@@ -32,8 +32,6 @@ module ThreeScale
 
     CONFIG_DELETE_STATS_BATCH_SIZE = 50
     private_constant :CONFIG_DELETE_STATS_BATCH_SIZE
-    CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE = 1000
-    private_constant :CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE
 
     @configuration = Configuration::Loader.new
 
@@ -54,13 +52,12 @@ module ThreeScale
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)
       config.add_section(:hoptoad, :service, :api_key)
-      config.add_section(:stats, :bucket_size, :delete_batch_size, :delete_partition_batch_size)
+      config.add_section(:stats, :bucket_size, :delete_batch_size)
       config.add_section(:redshift, :host, :port, :dbname, :user, :password)
       config.add_section(:statsd, :host, :port)
       config.add_section(:internal_api, :user, :password)
       config.add_section(:master, :metrics)
       config.add_section(:worker_prometheus_metrics, :enabled, :port)
-      config.add_section(:listener_prometheus_metrics, :enabled, :port)
 
       config.add_section(
           :async_worker,
@@ -125,9 +122,6 @@ module ThreeScale
       config.stats.delete_batch_size = parse_int(config.stats.delete_batch_size,
                                                  CONFIG_DELETE_STATS_BATCH_SIZE)
 
-      config.stats.delete_partition_batch_size = parse_int(config.stats.delete_partition_batch_size,
-                                                           CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE)
-
       # often we don't have a log_file setting - generate it here from
       # the log_path setting.
       log_file = config.log_file

data/lib/3scale/backend/errors.rb

@@ -292,12 +292,6 @@ module ThreeScale
       end
     end
 
-    class DeleteServiceStatsValidationError < Error
-      def initialize(service_id, msg)
-        super "Delete stats job context validation error. Service: #{service_id}. Error: #{msg}"
-      end
-    end
-
     class EndUsersNoLongerSupported < BadRequest
       def initialize
         super 'End-users are no longer supported, do not specify the user_id parameter'.freeze

data/lib/3scale/backend/rack.rb

@@ -17,7 +17,10 @@ module ThreeScale
 
           Backend::Logging::External.setup_rack self
 
-          if Backend.configuration.listener_prometheus_metrics.enabled
+          # Notice that this cannot be specified via config, it needs to be an
+          # ENV because the metric server is started in Puma/Falcon
+          # "before_fork" and the configuration is not loaded at that point.
+          if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s.downcase.freeze == 'true'.freeze
             use Rack::Prometheus
           end
 

data/lib/3scale/backend/service.rb

@@ -4,16 +4,12 @@ module ThreeScale
       include Storable
 
       # list of attributes to be fetched from storage
-      ATTRIBUTES = %i[state referrer_filters_required backend_version
-                      user_registration_required default_user_plan_id
-                      default_user_plan_name provider_key].freeze
+      ATTRIBUTES = %i[state referrer_filters_required backend_version provider_key].freeze
       private_constant :ATTRIBUTES
 
       attr_reader :state
-      attr_accessor :provider_key, :id, :backend_version,
-        :default_user_plan_id, :default_user_plan_name
-      attr_writer :referrer_filters_required, :user_registration_required,
-        :default_service
+      attr_accessor :provider_key, :id, :backend_version
+      attr_writer :referrer_filters_required, :default_service
 
       class << self
         include Memoizer::Decorator
@@ -104,8 +100,6 @@ module ThreeScale
         memoize :list
 
         def save!(attributes = {})
-          massage_set_user_registration_required attributes
-
           new(attributes).save!
         end
 
@@ -139,26 +133,10 @@ module ThreeScale
         def massage_service_attrs(service_attrs)
           service_attrs[:referrer_filters_required] =
             service_attrs[:referrer_filters_required].to_i > 0
-          service_attrs[:user_registration_required] =
-            massage_get_user_registration_required(
-              service_attrs[:user_registration_required])
 
           service_attrs
         end
 
-        # nil => true, 1 => true, '1' => true, 0 => false, '0' => false
-        def massage_get_user_registration_required(value)
-          value.nil? ? true : value.to_i > 0
-        end
-
-        def massage_set_user_registration_required(attributes)
-          if attributes[:user_registration_required].nil?
-            val = storage.get(storage_key(attributes[:id], :user_registration_required))
-            attributes[:user_registration_required] =
-              (!val.nil? && val.to_i == 0) ? false : true
-          end
-        end
-
         def get_attr(id, attribute)
           storage.get(storage_key(id, attribute))
         end
@@ -195,10 +173,6 @@ module ThreeScale
         @referrer_filters_required
       end
 
-      def user_registration_required?
-        @user_registration_required
-      end
-
       def save!
         set_as_default_if_needed
         persist
@@ -227,9 +201,6 @@ module ThreeScale
           provider_key: provider_key,
           backend_version: backend_version,
           referrer_filters_required: referrer_filters_required?,
-          user_registration_required: user_registration_required?,
-          default_user_plan_id: default_user_plan_id,
-          default_user_plan_name: default_user_plan_name,
           default_service: default_service?
         }
       end
@@ -294,9 +265,6 @@ module ThreeScale
 
       def persist_attributes
         persist_attribute :referrer_filters_required, referrer_filters_required? ? 1 : 0
-        persist_attribute :user_registration_required, user_registration_required? ? 1 : 0
-        persist_attribute :default_user_plan_id, default_user_plan_id, true
-        persist_attribute :default_user_plan_name, default_user_plan_name, true
         persist_attribute :backend_version, backend_version, true
         persist_attribute :provider_key, provider_key
         persist_attribute :state, state.to_s if state

data/lib/3scale/backend/stats.rb

@@ -1,8 +1,4 @@
 require '3scale/backend/stats/codes_commons'
 require '3scale/backend/stats/period_commons'
 require '3scale/backend/stats/aggregator'
-require '3scale/backend/stats/delete_job_def'
-require '3scale/backend/stats/key_generator'
-require '3scale/backend/stats/partition_generator_job'
-require '3scale/backend/stats/partition_eraser_job'
 require '3scale/backend/stats/cleaner'

data/lib/3scale/backend/stats/aggregator.rb

@@ -145,6 +145,16 @@ module ThreeScale
               application = Backend::Application.load(service_id,
                                                       values[:application_id])
 
+              # The app could have been deleted at some point since the job was
+              # enqueued. No need to update alerts in that case.
+              next unless application
+
+              # The operations below are costly. They load all the usage limits
+              # and current usages to find the current utilization levels.
+              # That's why before that, we check if there are any alerts that
+              # can be raised.
+              next unless Alerts.can_raise_more_alerts?(service_id, values[:application_id])
+
               application.load_metric_names
               usage = Usage.application_usage(application, current_timestamp)
               status = Transactor::Status.new(service_id: service_id,

data/lib/3scale/backend/stats/aggregators/base.rb

@@ -20,7 +20,14 @@ module ThreeScale
                 key = counter_key(prefix_key, granularity.new(timestamp))
                 expire_time = Stats::PeriodCommons.expire_time_for_granularity(granularity)
 
-                store_key(cmd, key, value, expire_time)
+                # We don't need to store stats keys set to 0. It wastes Redis
+                # memory because for rate-limiting and stats, a key of set to 0
+                # is equivalent to a key that does not exist.
+                if cmd == :set && value == 0
+                  storage.del(key)
+                else
+                  store_key(cmd, key, value, expire_time)
+                end
 
                 unless Stats::PeriodCommons::EXCLUDED_FOR_BUCKETS.include?(granularity)
                   keys_for_bucket << key

data/lib/3scale/backend/stats/period_commons.rb

@@ -12,9 +12,6 @@ module ThreeScale
         GRANULARITY_EXPIRATION_TIME = { Period[:minute] => 180 }.freeze
         private_constant :GRANULARITY_EXPIRATION_TIME
 
-        PERMANENT_SERVICE_GRANULARITIES = (SERVICE_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-        PERMANENT_EXPANDED_GRANULARITIES = (EXPANDED_GRANULARITIES - GRANULARITY_EXPIRATION_TIME.keys).freeze
-
         # We are not going to send metrics with granularity 'eternity' or
         # 'week' to Kinesis, so there is no point in storing them in Redis
         # buckets.

data/lib/3scale/backend/transactor.rb

@@ -61,6 +61,8 @@ module ThreeScale
 
       def validate(oauth, provider_key, report_usage, params, request_info)
         service = Service.load_with_provider_key!(params[:service_id], provider_key)
+        oidc_service = !oauth && service.backend_version == 'oauth'.freeze
+
         # service_id cannot be taken from params since it might be missing there
         service_id = service.id
 
@@ -70,12 +72,18 @@ module ThreeScale
         # significant.
         params[:app_id] = nil if app_id && app_id.empty?
 
-        if oauth
-          raise ApplicationNotFound.new nil if app_id.nil?
-          validators = Validators::OAUTH_VALIDATORS
-        else
-          validators = Validators::VALIDATORS
-        end
+        # While OIDC without an app_id makes little sense, we would break existing
+        # behaviour when calling non oauth_auth*.xml endpoints if we returned an
+        # error here, so only do this for oauth_auth*.xml endpoints.
+        raise ApplicationNotFound.new nil if oauth && app_id.nil?
+
+        validators = if oidc_service
+                       Validators::OIDC_VALIDATORS
+                     elsif oauth
+                       Validators::OAUTH_VALIDATORS
+                     else
+                       Validators::VALIDATORS
+                     end
 
         params[:user_key] = nil if params[:user_key] && params[:user_key].empty?
         application = Application.load_by_id_or_user_key!(service_id,
@@ -98,8 +106,9 @@ module ThreeScale
           # hierarchy parameter adds information in the response needed
           # to derive which limits affect directly or indirectly the
           # metrics for which authorization is requested.
-          hierarchy:       extensions[:hierarchy] == '1',
-          flat_usage:      extensions[:flat_usage] == '1'
+          hierarchy:       extensions[:hierarchy] == '1'.freeze,
+          flat_usage:      extensions[:flat_usage] == '1'.freeze,
+          list_app_keys:   extensions[:list_app_keys] == '1'.freeze
         }
 
         application.load_metric_names
@@ -108,24 +117,6 @@ module ThreeScale
         apply_validators(validators, status_attrs, params)
       end
 
-      def get_token_ids(token, service_id, app_id)
-        begin
-          token_aid = OAuth::Token::Storage.get_credentials(token, service_id)
-        rescue AccessTokenInvalid => e
-          # Yep, well, er. Someone specified that it is OK to have an
-          # invalid token if an app_id is specified. Somehow passing in
-          # a user_key is still not enough, though...
-          raise e if app_id.nil?
-        end
-
-        # We only take the token ids into account if we had no parameter ids
-        if app_id.nil?
-          app_id = token_aid
-        end
-
-        app_id
-      end
-
       def do_authorize(method, provider_key, params, context_info)
         notify_authorize(provider_key)
         validate(method == :oauth_authorize, provider_key, false, params, context_info[:request])

data/lib/3scale/backend/transactor/status.rb

@@ -8,17 +8,23 @@ module ThreeScale
         # We only use 'redirect_uri' if a request sent such a param. See #397.
         REDIRECT_URI_FIELD = 'redirect_url'.freeze
         private_constant :REDIRECT_URI_FIELD
+        # Maximum number of keys to list when using the list_app_keys extension
+        # At the time of writing System/Porta has a limit of 5 different app_keys
+        # at any given moment, but this could change anytime.
+        LIST_APP_KEYS_MAX = 256
+        private_constant :LIST_APP_KEYS_MAX
 
         def initialize(attributes)
-          @service_id      = attributes[:service_id]
-          @application     = attributes[:application]
-          @oauth           = attributes[:oauth]
-          @usage           = attributes[:usage]
-          @predicted_usage = attributes[:predicted_usage]
-          @values          = filter_values(attributes[:values] || {})
-          @timestamp       = attributes[:timestamp] || Time.now.getutc
-          @hierarchy_ext   = attributes[:hierarchy]
-          @flat_usage_ext  = attributes[:flat_usage]
+          @service_id        = attributes[:service_id]
+          @application       = attributes[:application]
+          @oauth             = attributes[:oauth]
+          @usage             = attributes[:usage]
+          @predicted_usage   = attributes[:predicted_usage]
+          @values            = filter_values(attributes[:values] || {})
+          @timestamp         = attributes[:timestamp] || Time.now.getutc
+          @hierarchy_ext     = attributes[:hierarchy]
+          @flat_usage_ext    = attributes[:flat_usage]
+          @list_app_keys_ext = attributes[:list_app_keys]
 
           raise 'service_id not specified' if @service_id.nil?
           raise ':application is required' if @application.nil?
@@ -106,6 +112,7 @@ module ThreeScale
             add_plan_section(xml, 'plan'.freeze, plan_name)
             add_reports_section(xml, application_usage_reports)
             hierarchy_reports.concat application_usage_reports if hierarchy_reports
+            add_app_keys_section xml if @list_app_keys_ext
           end
 
           if hierarchy_reports
@@ -161,6 +168,17 @@ module ThreeScale
           xml << '</hierarchy>'.freeze
         end
 
+        def add_app_keys_section(xml)
+          xml << '<app_keys app="'.freeze
+          xml << @application.id << '" svc="'.freeze
+          xml << @service_id << '">'.freeze
+          @application.keys.take(LIST_APP_KEYS_MAX).each do |key|
+            xml << '<key id="'.freeze
+            xml << key << '"/>'.freeze
+          end
+          xml << '</app_keys>'.freeze
+        end
+
         # helper to iterate over reports and get relevant hierarchy info
         def with_report_and_hierarchy(reports)
           reports.each do |ur|

data/lib/3scale/backend/validators.rb

@@ -21,6 +21,13 @@ module ThreeScale
       OAUTH_VALIDATORS = ([Validators::OauthSetting,
                            Validators::OauthKey,
                            Validators::RedirectURI] + COMMON_VALIDATORS).freeze
+
+      # OIDC specific validators will only check app keys when app_key is given.
+      #
+      # No need to add OauthSetting, since we need to check that to tell
+      # OIDC apart from the rest when calling authrep.xml (note lack of
+      # the oauth_ prefix).
+      OIDC_VALIDATORS = ([Validators::OauthKey] + COMMON_VALIDATORS).freeze
     end
   end
 end

data/lib/3scale/backend/validators/oauth_setting.rb

@@ -3,7 +3,7 @@ module ThreeScale
     module Validators
       class OauthSetting < Base
         def apply
-          if service.backend_version == 'oauth'
+          if service.backend_version == 'oauth'.freeze
             succeed!
           else
             fail!(OauthNotEnabled.new)

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.3.0'
+    VERSION = '3.4.0'
   end
 end

data/lib/3scale/prometheus_server.rb

@@ -4,7 +4,7 @@
 require_relative '../3scale/backend/listener_metrics'
 
 # Config is not loaded at this point, so read ENV instead.
-if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s == 'true'
+if ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_ENABLED'].to_s.downcase.freeze == 'true'.freeze
   prometheus_port = ENV['CONFIG_LISTENER_PROMETHEUS_METRICS_PORT']
   ThreeScale::Backend::ListenerMetrics.start_metrics_server(prometheus_port)
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.3.0</version>
+        <version>3.4.0</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -233,7 +233,7 @@
       </dependency>
           <dependency>
         <packageName>coderay</packageName>
-        <version>1.1.2</version>
+        <version>1.1.3</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -421,7 +421,7 @@
       </dependency>
           <dependency>
         <packageName>method_source</packageName>
-        <version>0.9.0</version>
+        <version>1.0.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -475,7 +475,7 @@
       </dependency>
           <dependency>
         <packageName>multi_json</packageName>
-        <version>1.13.1</version>
+        <version>1.15.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -679,7 +679,7 @@
       </dependency>
           <dependency>
         <packageName>pry</packageName>
-        <version>0.11.3</version>
+        <version>0.14.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -699,7 +699,7 @@
       </dependency>
           <dependency>
         <packageName>pry-doc</packageName>
-        <version>0.11.1</version>
+        <version>1.1.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -709,7 +709,7 @@
       </dependency>
           <dependency>
         <packageName>puma</packageName>
-        <version>2.15.3</version>
+        <version>4.3.7</version>
         <licenses>
                       <license>
               <name>New BSD</name>
@@ -1143,7 +1143,7 @@
       </dependency>
           <dependency>
         <packageName>yard</packageName>
-        <version>0.9.20</version>
+        <version>0.9.26</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-09 00:00:00.000000000 Z
+date: 2021-06-14 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -136,11 +136,7 @@ files:
 - lib/3scale/backend/stats/bucket_storage.rb
 - lib/3scale/backend/stats/cleaner.rb
 - lib/3scale/backend/stats/codes_commons.rb
-- lib/3scale/backend/stats/delete_job_def.rb
-- lib/3scale/backend/stats/key_generator.rb
 - lib/3scale/backend/stats/keys.rb
-- lib/3scale/backend/stats/partition_eraser_job.rb
-- lib/3scale/backend/stats/partition_generator_job.rb
 - lib/3scale/backend/stats/period_commons.rb
 - lib/3scale/backend/stats/stats_parser.rb
 - lib/3scale/backend/stats/storage.rb
@@ -211,8 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.7
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.8
+rubygems_version: 3.2.10
 signing_key: 
 specification_version: 4
 summary: 3scale web service management system backend

data/lib/3scale/backend/stats/delete_job_def.rb

@@ -1,60 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class DeleteJobDef
-        ATTRIBUTES = %i[service_id applications metrics from to context_info].freeze
-        private_constant :ATTRIBUTES
-        attr_reader(*ATTRIBUTES)
-
-        def self.attribute_names
-          ATTRIBUTES
-        end
-
-        def initialize(params = {})
-          ATTRIBUTES.each do |key|
-            instance_variable_set("@#{key}".to_sym, params[key]) unless params[key].nil?
-          end
-          validate
-        end
-
-        def run_async
-          Resque.enqueue(PartitionGeneratorJob, Time.now.getutc.to_f, service_id, applications,
-                         metrics, from, to, context_info)
-        end
-
-        def to_json
-          to_hash.to_json
-        end
-
-        def to_hash
-          Hash[ATTRIBUTES.collect { |key| [key, send(key)] }]
-        end
-
-        private
-
-        def validate
-          # from and to valid epoch times
-          raise_validation_error('from field not integer') unless from.is_a? Integer
-          raise_validation_error('from field is zero') if from.zero?
-          raise_validation_error('to field not integer') unless to.is_a? Integer
-          raise_validation_error('to field is zero') if to.zero?
-          raise_validation_error('from < to fields') if Time.at(to) < Time.at(from)
-          # application is array
-          raise_validation_error('applications field') unless applications.is_a? Array
-          raise_validation_error('applications values') unless applications.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-          # metrics is array
-          raise_validation_error('metrics field') unless metrics.is_a? Array
-          raise_validation_error('metrics values') unless metrics.all? do |x|
-            x.is_a?(String) || x.is_a?(Integer)
-          end
-        end
-
-        def raise_validation_error(msg)
-          raise DeleteServiceStatsValidationError.new(service_id, msg)
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/key_generator.rb

@@ -1,73 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      class KeyGenerator
-        attr_reader :service_id, :applications, :metrics, :from, :to
-
-        def initialize(service_id:, applications: [], metrics: [], from:, to:, **)
-          @service_id = service_id
-          @applications = applications
-          @metrics = metrics
-          @from = from
-          @to = to
-        end
-
-        def keys
-          response_code_service_keys +
-            response_code_application_keys +
-            usage_service_keys +
-            usage_application_keys
-        end
-
-        private
-
-        def periods(granularities)
-          granularities.flat_map do |granularity|
-            (Period[granularity].new(Time.at(from))..Period[granularity].new(Time.at(to))).to_a
-          end
-        end
-
-        def response_codes
-          CodesCommons::TRACKED_CODES + CodesCommons::TRACKED_CODE_GROUPS
-        end
-
-        def response_code_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              Keys.service_response_code_value_key(service_id, response_code, period)
-            end
-          end
-        end
-
-        def response_code_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            response_codes.flat_map do |response_code|
-              applications.flat_map do |application|
-                Keys.application_response_code_value_key(service_id, application,
-                                                         response_code, period)
-              end
-            end
-          end
-        end
-
-        def usage_service_keys
-          periods(PeriodCommons::PERMANENT_SERVICE_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              Keys.service_usage_value_key(service_id, metric, period)
-            end
-          end
-        end
-
-        def usage_application_keys
-          periods(PeriodCommons::PERMANENT_EXPANDED_GRANULARITIES).flat_map do |period|
-            metrics.flat_map do |metric|
-              applications.flat_map do |application|
-                Keys.application_usage_value_key(service_id, application, metric, period)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_eraser_job.rb

@@ -1,58 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Perform actual key deletion from a key partition definition
-      class PartitionEraserJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include StorageHelpers
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, offset, length, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            validate_job(job, offset, length)
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            stats_key_gen.keys.drop(offset).take(length).each_slice(configuration.stats.delete_batch_size) do |slice|
-              storage.del(slice)
-            end
-
-            [true, { job: job.to_hash, offset: offset, lenght: length }.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def validate_job(job, offset, length)
-            unless offset.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'offset field value ' \
-                                                          "[#{offset}] validation error")
-            end
-
-            unless length.is_a? Integer
-              raise DeleteServiceStatsValidationError.new(job.service_id, 'length field value ' \
-                                                          "[#{length}] validation error")
-            end
-          end
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/stats/partition_generator_job.rb

@@ -1,46 +0,0 @@
-module ThreeScale
-  module Backend
-    module Stats
-      # Job for deleting service stats
-      # Maps delete job definition to a set of non overlapping key set partitions
-      class PartitionGeneratorJob < BackgroundJob
-        # low priority queue
-        @queue = :stats
-
-        class << self
-          include Configurable
-
-          def perform_logged(_enqueue_time, service_id, applications, metrics,
-                             from, to, context_info = {})
-            job = DeleteJobDef.new(
-              service_id: service_id,
-              applications: applications,
-              metrics: metrics,
-              from: from,
-              to: to
-            )
-
-            stats_key_gen = KeyGenerator.new(job.to_hash)
-
-            # Generate partitions
-            0.step(stats_key_gen.keys.count, configuration.stats.delete_partition_batch_size).each do |idx|
-              Resque.enqueue(PartitionEraserJob, Time.now.getutc.to_f, service_id, applications,
-                             metrics, from, to, idx,
-                             configuration.stats.delete_partition_batch_size, context_info)
-            end
-
-            [true, job.to_json]
-          rescue Backend::Error => error
-            [false, "#{service_id} #{error}"]
-          end
-
-          private
-
-          def enqueue_time(args)
-            args[0]
-          end
-        end
-      end
-    end
-  end
-end