apisonator 2.100.0 → 2.101.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '02689c2d07caa6e88eb1079c13064378fafa921ffb28e4da095cd8ad5fbac578'
-  data.tar.gz: 434d69fc1eb06088b1c4111bd41979567f1b00c22b149e5b4a7bdbda48c447a1
+  metadata.gz: 4740b1613c8bd2182adf9587db2a2ae6667a50ee140c99c679c232b6da3b7193
+  data.tar.gz: 0e34376e2e788194b44eeb5aedf7d8654110c8baa4a0885ed2c55a3a28fcb0c1
 SHA512:
-  metadata.gz: edc21e4dedc90447e734e7afe0fe681127d9a4a88cd4ba3d9f9633b7eae3ee31488aa54b9cfaaeb6e28c158eb822c9679e325c4b6773130779fd63f5a6498fe9
-  data.tar.gz: 3267cfad364babfafb0c563feea99ecd0ff346b7a12d224b37178ff28e13a92fcce8a25372b1901f54a1bfcf6ecdeea0f9665ac2e30dcc6fe0fa3aff15bce468
+  metadata.gz: c6431c06f07433710308ef8b9121d2975415cbec27d2c82db3c8f13c4a0dab364b6a9085a550a530bd000a18234bfd63d67ee555ea69299f8745734725f9f436
+  data.tar.gz: 90618d349784bfe5ff9d78b940f93d7b63013fb9f1ec1b9c68f03881ff9e13a165d0532a98d50c26218402cc197c3f9f730f7b34ca26589cb8f4b91fa37e6cab

data/CHANGELOG.md

@@ -2,6 +2,46 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 2.101.0 - 2020-06-04
+
+### Added
+
+- Introduced the `CONFIG_REDIS_MAX_CONNS` and `CONFIG_QUEUES_MAX_CONNS` ENVs to
+configure the max number of Redis connections when using the async mode
+([#214](https://github.com/3scale/apisonator/pull/214)).
+
+### Changed
+
+- Perf optimization: loading the usage limits is now done more efficiently.
+There is a noticeable improvement in requests with `no_body` enabled for
+services with many metrics defined
+([#221](https://github.com/3scale/apisonator/pull/221)).
+- Updated activesupport to 5.2.4.3
+([#217](https://github.com/3scale/apisonator/pull/217)).
+
+
+## 2.100.2 - 2020-05-08
+
+### Changed
+
+- The Prometheus histogram buckets of the workers have been adjusted to be more
+informative ([#208](https://github.com/3scale/apisonator/pull/208)).
+
+### Removed
+
+- The deprecated endpoints to create, delete, and list oauth tokens have been
+disabled ([#212](https://github.com/3scale/apisonator/pull/212)).
+
+
+## 2.100.1 - 2020-04-22
+
+### Changed
+
+- Now we are using our own redis-rb fork. It includes a fix that should reduce
+the number of 5xx errors caused by Redis connection errors
+([#205](https://github.com/3scale/apisonator/pull/205)).
+- Updated hiredis to v0.6.3 ([#204](https://github.com/3scale/apisonator/pull/204)).
+
 ## 2.100.0 - 2020-04-21
 
 ### Added

data/Gemfile.base

@@ -9,7 +9,7 @@ gemspec
 # implementations (ie. pure Ruby, java, etc).
 #
 platform :ruby do
-  gem 'hiredis', '= 0.6.1'
+  gem 'hiredis', '~> 0.6.1'
   gem 'yajl-ruby', '~> 1.3.1', require: 'yajl'
   gem 'pry-byebug', '~> 3.5.1', groups: [:development]
 end
@@ -25,7 +25,6 @@ group :test do
   gem 'resque_spec',   '~> 0.17.0'
   gem 'timecop',       '~> 0.9.1'
   gem 'rspec',         '~> 3.7.0', require: nil
-  gem 'geminabox',     '~> 0.13.11', require: false
   gem 'codeclimate-test-reporter', '~> 0.6.0', require: nil
   gem 'async-rspec'
 end
@@ -52,7 +51,6 @@ gem 'daemons', '= 1.2.4'
 # Production gems
 gem 'rake', '~> 13.0'
 gem 'builder', '= 3.2.3'
-gem 'redis', '= 4.1.1'
 # Use a patched resque to allow reusing their Airbrake Failure class
 gem 'resque', git: 'https://github.com/3scale/resque', branch: '3scale'
 gem 'rack', '~> 2.0.8'
@@ -63,3 +61,8 @@ gem 'bugsnag', '~> 6', require: nil
 gem 'yabeda-prometheus', '~> 0.5.0'
 gem 'async-redis', '~> 0.4.1'
 gem 'falcon', '~> 0.35'
+
+# Use a patched redis-rb that fixes an issue when trying to connect with
+# sentinels and avoids retrying calls when there's a timeout to prevent
+# duplicated commands. It's based on version 4.1.3.
+gem 'redis', git: 'https://github.com/3scale/redis-rb', branch: 'apisonator'

data/Gemfile.lock

@@ -5,6 +5,13 @@ GIT
   specs:
     puma (2.15.3)
 
+GIT
+  remote: https://github.com/3scale/redis-rb
+  revision: 35301b3d952975300a4cb30d408ae3b5969d0440
+  branch: apisonator
+  specs:
+    redis (4.1.3)
+
 GIT
   remote: https://github.com/3scale/resque
   revision: 88839e71756ea9b6edfc9426a0af71e94109c135
@@ -28,14 +35,14 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (2.100.0)
+    apisonator (2.101.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.2.4.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     airbrake (4.3.1)
@@ -88,7 +95,7 @@ GEM
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
     coderay (1.1.2)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
     diff-lcs (1.3)
@@ -105,20 +112,10 @@ GEM
       process-metrics (~> 0.1.0)
       rack (>= 1.0)
       samovar (~> 2.1)
-    faraday (0.13.1)
-      multipart-post (>= 1.2, < 3)
     ffi (1.12.2)
-    geminabox (0.13.11)
-      builder
-      faraday
-      httpclient (>= 2.2.7)
-      nesty
-      reentrant_flock
-      sinatra (>= 1.2.7)
     gli (2.16.1)
-    hiredis (0.6.1)
-    httpclient (2.8.3)
-    i18n (0.9.1)
+    hiredis (0.6.3)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jmespath (1.3.1)
     json (2.1.0)
@@ -134,15 +131,13 @@ GEM
     metaclass (0.0.4)
     method_source (0.9.0)
     mini_portile2 (2.4.0)
-    minitest (5.10.3)
+    minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
     multi_json (1.13.1)
-    multipart-post (2.0.0)
     mustache (1.0.5)
     mustermann (1.0.2)
-    nesty (1.0.2)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
@@ -182,10 +177,8 @@ GEM
     rack-test (0.8.2)
       rack (>= 1.0, < 3)
     rake (13.0.1)
-    redis (4.1.1)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
-    reentrant_flock (0.1.1)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
       rspec-core (>= 3.0.0)
@@ -250,7 +243,7 @@ GEM
     timers (4.3.0)
     toml (0.2.0)
       parslet (~> 1.8.0)
-    tzinfo (1.2.4)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     vegas (0.1.11)
       rack (>= 1.0.0)
@@ -282,9 +275,8 @@ DEPENDENCIES
   codeclimate-test-reporter (~> 0.6.0)
   daemons (= 1.2.4)
   falcon (~> 0.35)
-  geminabox (~> 0.13.11)
   gli (~> 2.16.1)
-  hiredis (= 0.6.1)
+  hiredis (~> 0.6.1)
   license_finder (~> 5)
   mocha (~> 1.3)
   nokogiri (~> 1.10.8)
@@ -297,7 +289,7 @@ DEPENDENCIES
   rack (~> 2.0.8)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
-  redis (= 4.1.1)
+  redis!
   resque!
   resque_spec (~> 0.17.0)
   resque_unit (~> 0.4.4)!

data/Gemfile.on_prem.lock

@@ -5,6 +5,13 @@ GIT
   specs:
     puma (2.15.3)
 
+GIT
+  remote: https://github.com/3scale/redis-rb
+  revision: 35301b3d952975300a4cb30d408ae3b5969d0440
+  branch: apisonator
+  specs:
+    redis (4.1.3)
+
 GIT
   remote: https://github.com/3scale/resque
   revision: 88839e71756ea9b6edfc9426a0af71e94109c135
@@ -28,14 +35,14 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (2.100.0)
+    apisonator (2.101.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.2.4.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     async (1.24.2)
@@ -78,7 +85,7 @@ GEM
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
     coderay (1.1.2)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
     diff-lcs (1.3)
@@ -95,20 +102,10 @@ GEM
       process-metrics (~> 0.1.0)
       rack (>= 1.0)
       samovar (~> 2.1)
-    faraday (0.13.1)
-      multipart-post (>= 1.2, < 3)
     ffi (1.12.2)
-    geminabox (0.13.11)
-      builder
-      faraday
-      httpclient (>= 2.2.7)
-      nesty
-      reentrant_flock
-      sinatra (>= 1.2.7)
     gli (2.16.1)
-    hiredis (0.6.1)
-    httpclient (2.8.3)
-    i18n (0.9.1)
+    hiredis (0.6.3)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     json (2.1.0)
     license_finder (5.9.2)
@@ -123,15 +120,13 @@ GEM
     metaclass (0.0.4)
     method_source (0.9.0)
     mini_portile2 (2.4.0)
-    minitest (5.10.3)
+    minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
     multi_json (1.13.1)
-    multipart-post (2.0.0)
     mustache (1.0.5)
     mustermann (1.0.2)
-    nesty (1.0.2)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
@@ -170,10 +165,8 @@ GEM
     rack-test (0.8.2)
       rack (>= 1.0, < 3)
     rake (13.0.1)
-    redis (4.1.1)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
-    reentrant_flock (0.1.1)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
       rspec-core (>= 3.0.0)
@@ -236,7 +229,7 @@ GEM
     timers (4.3.0)
     toml (0.2.0)
       parslet (~> 1.8.0)
-    tzinfo (1.2.4)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     vegas (0.1.11)
       rack (>= 1.0.0)
@@ -264,9 +257,8 @@ DEPENDENCIES
   codeclimate-test-reporter (~> 0.6.0)
   daemons (= 1.2.4)
   falcon (~> 0.35)
-  geminabox (~> 0.13.11)
   gli (~> 2.16.1)
-  hiredis (= 0.6.1)
+  hiredis (~> 0.6.1)
   license_finder (~> 5)
   mocha (~> 1.3)
   nokogiri (~> 1.10.8)
@@ -278,7 +270,7 @@ DEPENDENCIES
   rack (~> 2.0.8)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
-  redis (= 4.1.1)
+  redis!
   resque!
   resque_spec (~> 0.17.0)
   resque_unit (~> 0.4.4)!

data/lib/3scale/backend.rb

@@ -5,7 +5,6 @@ require 'builder'
 require 'hiredis'
 
 require 'redis'
-require '3scale/backend/extensions/redis'
 
 require 'resque'
 require 'securerandom'

data/lib/3scale/backend/application.rb

@@ -223,6 +223,24 @@ module ThreeScale
         @usage_limits ||= UsageLimit.load_all(service_id, plan_id)
       end
 
+      def load_all_usage_limits
+        @usage_limits = UsageLimit.load_all(service_id, plan_id)
+      end
+
+      # Loads the usage limits affected by the metrics received, that is, the
+      # limits that are defined for those metrics plus all their ancestors in
+      # the metrics hierarchy.
+      def load_usage_limits_affected_by(metric_names)
+        metric_ids = metric_names.flat_map do |name|
+          [name] + Metric.ascendants(service_id, name)
+        end.uniq.map do |name|
+          Metric.load_id(service_id, name)
+        end
+
+        # IDs are sorted to be able to use the memoizer
+        @usage_limits = UsageLimit.load_for_affecting_metrics(service_id, plan_id, metric_ids.sort)
+      end
+
       def active?
         state == :active
       end

data/lib/3scale/backend/configuration.rb

@@ -46,9 +46,9 @@ module ThreeScale
 
       # Add configuration sections
       config.add_section(:queues, :master_name, :sentinels, :role,
-                         :connect_timeout, :read_timeout, :write_timeout)
+                         :connect_timeout, :read_timeout, :write_timeout, :max_connections)
       config.add_section(:redis, :url, :proxy, :sentinels, :role,
-                         :connect_timeout, :read_timeout, :write_timeout,
+                         :connect_timeout, :read_timeout, :write_timeout, :max_connections,
                          :async)
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)

data/lib/3scale/backend/listener.rb

@@ -12,9 +12,9 @@ module ThreeScale
       include Logging
 
       ## ------------ DOCS --------------
-      ##~ namespace = ENV['SAAS_SWAGGER'] == "1" ? "Service Management API" : "Service Management API (on-premises)"
+      ##~ namespace = "Service Management API"
       ##~ sapi = source2swagger.namespace(namespace)
-      ##~ sapi.basePath = "https://su1.3scale.net"
+      ##~ sapi.basePath = ""
       ##~ sapi.swaggerVersion = "0.1a"
       ##~ sapi.apiVersion = "1.0"
       ##
@@ -190,7 +190,7 @@ module ThreeScale
       private :do_api_method
 
       ## ------------ DOCS --------------
-      ##~ namespace = ENV['SAAS_SWAGGER'] == "1" ? "Service Management API" : "Service Management API (on-premises)"
+      ##~ namespace = "Service Management API"
       ##~ sapi = source2swagger.namespace(namespace)
       ##~ a = sapi.apis.add
       ##~ a.set "path" => "/transactions/authorize.xml", "format" => "xml"
@@ -243,7 +243,7 @@ module ThreeScale
       end
 
       ## ------------ DOCS --------------
-      ##~ namespace = ENV['SAAS_SWAGGER'] == "1" ? "Service Management API" : "Service Management API (on-premises)"
+      ##~ namespace = "Service Management API"
       ##~ sapi = source2swagger.namespace(namespace)
       ##~ a = sapi.apis.add
       ##~ a.set "path" => "/transactions/oauth_authorize.xml", "format" => "xml"
@@ -276,7 +276,7 @@ module ThreeScale
       end
 
       ## ------------ DOCS --------------
-      ##~ namespace = ENV['SAAS_SWAGGER'] == "1" ? "Service Management API" : "Service Management API (on-premises)"
+      ##~ namespace = "Service Management API"
       ##~ sapi = source2swagger.namespace(namespace)
       ##~ a = sapi.apis.add
       ##~ a.set "path" => "/transactions/authrep.xml", "format" => "xml"
@@ -323,7 +323,7 @@ module ThreeScale
       end
 
       ## ------------ DOCS --------------
-      ##~ namespace = ENV['SAAS_SWAGGER'] == "1" ? "Service Management API" : "Service Management API (on-premises)"
+      ##~ namespace = "Service Management API"
       ##~ sapi = source2swagger.namespace(namespace)
       ##~ a = sapi.apis.add
       ##~ a.set "path" => "/transactions/oauth_authrep.xml", "format" => "xml"
@@ -351,7 +351,7 @@ module ThreeScale
       end
 
       ## ------------ DOCS --------------
-      ##~ namespace = ENV['SAAS_SWAGGER'] == "1" ? "Service Management API" : "Service Management API (on-premises)"
+      ##~ namespace = "Service Management API"
       ##~ sapi = source2swagger.namespace(namespace)
       ##~ a = sapi.apis.add
       ##~ a.set "path" => "/transactions.xml", "format" => "xml"
@@ -432,54 +432,58 @@ module ThreeScale
 
       ## OAUTH ACCESS TOKENS
 
-      post '/services/:service_id/oauth_access_tokens.xml' do
-        check_post_content_type!
-        require_params! :service_id, :token
+      # These endpoints are deprecated and are going to be removed. For now,
+      # let's disable them.
+      if Backend.test?
+        post '/services/:service_id/oauth_access_tokens.xml' do
+          check_post_content_type!
+          require_params! :service_id, :token
 
-        service_id = params[:service_id]
-        ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
+          service_id = params[:service_id]
+          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
 
-        app_id = params[:app_id]
-        raise ApplicationNotFound, app_id unless Application.exists?(service_id, app_id)
+          app_id = params[:app_id]
+          raise ApplicationNotFound, app_id unless Application.exists?(service_id, app_id)
 
-        OAuth::Token::Storage.create(params[:token], service_id, app_id, params[:ttl])
-      end
+          OAuth::Token::Storage.create(params[:token], service_id, app_id, params[:ttl])
+        end
 
-      delete '/services/:service_id/oauth_access_tokens/:token.xml' do
-        require_params! :service_id, :token
+        delete '/services/:service_id/oauth_access_tokens/:token.xml' do
+          require_params! :service_id, :token
 
-        service_id = params[:service_id]
-        ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
+          service_id = params[:service_id]
+          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
 
-        token = params[:token]
+          token = params[:token]
 
-        # TODO: perhaps improve this to list the deleted tokens?
-        raise AccessTokenInvalid, token unless OAuth::Token::Storage.delete(token, service_id)
-      end
+          # TODO: perhaps improve this to list the deleted tokens?
+          raise AccessTokenInvalid, token unless OAuth::Token::Storage.delete(token, service_id)
+        end
 
-      get '/services/:service_id/applications/:app_id/oauth_access_tokens.xml' do
-        require_params! :service_id, :app_id
+        get '/services/:service_id/applications/:app_id/oauth_access_tokens.xml' do
+          require_params! :service_id, :app_id
 
-        service_id = params[:service_id]
-        ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
+          service_id = params[:service_id]
+          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
 
-        app_id = params[:app_id]
+          app_id = params[:app_id]
 
-        raise ApplicationNotFound, app_id unless Application.exists?(service_id, app_id)
+          raise ApplicationNotFound, app_id unless Application.exists?(service_id, app_id)
 
-        @tokens = OAuth::Token::Storage.all_by_service_and_app service_id, app_id
-        builder :oauth_access_tokens
-      end
+          @tokens = OAuth::Token::Storage.all_by_service_and_app service_id, app_id
+          builder :oauth_access_tokens
+        end
 
-      get '/services/:service_id/oauth_access_tokens/:token.xml' do
-        require_params! :service_id, :token
+        get '/services/:service_id/oauth_access_tokens/:token.xml' do
+          require_params! :service_id, :token
 
-        service_id = params[:service_id]
-        ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
+          service_id = params[:service_id]
+          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
 
-        @token_to_app_id = OAuth::Token::Storage.get_credentials(params[:token], service_id)
+          @token_to_app_id = OAuth::Token::Storage.get_credentials(params[:token], service_id)
 
-        builder :oauth_app_id_by_token
+          builder :oauth_app_id_by_token
+        end
       end
 
       get '/check.txt' do
@@ -600,10 +604,6 @@ module ThreeScale
         end
       end
 
-      def application
-        @application ||= Application.load_by_id_or_user_key!(service_id, params[:app_id], params[:user_key])
-      end
-
       def service_id
         if params[:service_id].nil? || params[:service_id].empty?
           @service_id ||= Service.default_id!(params[:provider_key])

data/lib/3scale/backend/metric/collection.rb

@@ -75,10 +75,7 @@ module ThreeScale
         end
 
         def load_metric_id(name)
-          Memoizer.memoize_block(Memoizer.build_key(self,
-                                        :load_metric_id, @service_id, name)) do
-            storage.get(encode_key("metric/service_id:#{@service_id}/name:#{name}/id"))
-          end || raise(MetricInvalid.new(name))
+          Metric.load_id(@service_id, name) || raise(MetricInvalid.new(name))
         end
 
         ## accepts postive integers or positive integers preffixed with # (for sets)

data/lib/3scale/backend/storage_async/client.rb

@@ -46,7 +46,9 @@ module ThreeScale
           port ||= DEFAULT_PORT
 
           endpoint = Async::IO::Endpoint.tcp(host, port)
-          @redis_async = Async::Redis::Client.new(endpoint)
+          @redis_async = Async::Redis::Client.new(
+            endpoint, connection_limit: opts[:max_connections]
+          )
           @building_pipeline = false
         end
 

data/lib/3scale/backend/storage_helpers.rb

@@ -44,17 +44,26 @@ module ThreeScale
               connect_timeout: 5,
               read_timeout: 3,
               write_timeout: 3,
-              # this is set to zero to avoid potential double transactions
-              # see https://github.com/redis/redis-rb/issues/668
-              reconnect_attempts: 0,
+              # Note that we can set reconnect_attempts to >= 0 because we use
+              # our redis-rb fork which implements a workaround for this issue
+              # that shows that when there might be duplicated transactions when
+              # there's a timeout: https://github.com/redis/redis-rb/issues/668
+              # We should investigate if there are edge cases that can lead to
+              # duplicated commands because of this setting.
+              reconnect_attempts: 1,
               # use by default the C extension client
-              driver: :hiredis
+              driver: :hiredis,
+              # applies only to async mode. The sync library opens 1 connection
+              # per process.
+              max_connections: 10,
           }.freeze
           private_constant :CONN_OPTIONS
 
           # CONN_WHITELIST - Connection options that can be specified in config
           # Note: we don't expose reconnect_attempts until the bug above is fixed
-          CONN_WHITELIST = [:connect_timeout, :read_timeout, :write_timeout].freeze
+          CONN_WHITELIST = [
+            :connect_timeout, :read_timeout, :write_timeout, :max_connections
+          ].freeze
           private_constant :CONN_WHITELIST
 
           # Parameters regarding target server we will take from a config object

data/lib/3scale/backend/transactor.rb

@@ -87,9 +87,13 @@ module ThreeScale
         application = Application.load_by_id_or_user_key!(service_id,
                                                           app_id,
                                                           params[:user_key])
+
+        extensions = request_info && request_info[:extensions] || {}
+
+        preload_usage_limits(application, extensions[:no_body], params[:usage])
+
         now          = Time.now.getutc
         usage_values = Usage.application_usage(application, now)
-        extensions   = request_info && request_info[:extensions] || {}
         status_attrs = {
           service_id:      service_id,
           application:     application,
@@ -169,6 +173,27 @@ module ThreeScale
         Resque.enqueue(ReportJob, service_id, data, Time.now.getutc.to_f, context_info)
       end
 
+      # Loads the usage limits that are needed to authorize the current request.
+      # These are the cases:
+      # - When no_body is enabled, we only need to load the limits related with
+      # the metrics included in the request, that is, the ones included plus all
+      # their ancestors in the hierarchy. That's all we need to verify if the
+      # request is within the limits defined.
+      # - When no_body is disabled, we need to load all the limits because they
+      # are needed to generate the XML response.
+      # - When the usage reported in the request is empty, apisonator returns
+      # "limits_exceeded" if any of the limits defined has been violated. That's
+      # why in that scenario we need to load all the limits.
+      def preload_usage_limits(application, no_body_enabled, usage_in_params)
+        metric_names_in_usage = usage_in_params&.keys || {}
+
+        if no_body_enabled && !metric_names_in_usage.empty?
+          application.load_usage_limits_affected_by(metric_names_in_usage)
+        else
+          application.load_all_usage_limits
+        end
+      end
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/transactor/status.rb

@@ -80,11 +80,6 @@ module ThreeScale
           values && values[usage_limit.metric_id] || 0
         end
 
-        def value_for_application_usage_limit(usage_limit)
-          values = @values[usage_limit.period]
-          values && values[usage_limit.metric_id] || 0
-        end
-
         # provides a hierarchy hash with metrics as symbolic names
         def hierarchy
           @hierarchy ||= Metric.hierarchy service_id

data/lib/3scale/backend/usage_limit.rb

@@ -22,20 +22,15 @@ module ThreeScale
 
         def load_all(service_id, plan_id)
           metric_ids = Metric.load_all_ids(service_id)
-          return metric_ids if metric_ids.empty?
-
-          results = []
-          with_pairs_and_values service_id, plan_id, metric_ids do |pair, value|
-            value and results << new(service_id: service_id,
-                                     plan_id: plan_id,
-                                     metric_id: pair[0],
-                                     period: pair[1],
-                                     value: value.to_i)
-          end
-          results
+          generate_for_metrics(service_id, plan_id, metric_ids)
         end
         memoize :load_all
 
+        def load_for_affecting_metrics(service_id, plan_id, metric_ids)
+          generate_for_metrics(service_id, plan_id, metric_ids)
+        end
+        memoize :load_for_affecting_metrics
+
         def load_value(service_id, plan_id, metric_id, period)
           raw_value = storage.get(key(service_id, plan_id, metric_id, period))
           raw_value and raw_value.to_i
@@ -80,6 +75,20 @@ module ThreeScale
           encode_key(key_pre + period.to_s)
         end
 
+        def generate_for_metrics(service_id, plan_id, metric_ids)
+          return metric_ids if metric_ids.empty?
+
+          results = []
+          with_pairs_and_values service_id, plan_id, metric_ids do |pair, value|
+            value and results << new(service_id: service_id,
+                                     plan_id: plan_id,
+                                     metric_id: pair[0],
+                                     period: pair[1],
+                                     value: value.to_i)
+          end
+          results
+        end
+
         # yields [pair(metric_id, period), value]
         def with_pairs_and_values(service_id, plan_id, metric_ids, &blk)
           pairs, values = get_pairs_and_values_for service_id, plan_id, metric_ids

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '2.100.0'
+    VERSION = '2.101.0'
   end
 end

data/lib/3scale/backend/worker_metrics.rb

@@ -11,7 +11,8 @@ Yabeda.configure do
       comment "How long jobs take to run"
       unit :seconds
       tags %i[type]
-      buckets [0.005, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]
+      # Most requests will be under 100ms, so use a higher granularity from there
+      buckets [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
     end
   end
 end

data/lib/3scale/tasks/swagger.rake

@@ -1,46 +1,28 @@
+SPEC_FILE = 'docs/active_docs/Service Management API.json'
+
 def run_command cmd
   puts "--> executing: #{cmd}"
   puts "--> Remember to commit the resulting specs to the Porta repo."
   system cmd
 end
 
-# The file name depends on the 'namespace' defined in the Listener.
-def spec_file(type)
-  res = 'docs/active_docs/Service Management API'
-  res << ' (on-premises)' if type == :on_prem
-  "#{res}.json"
-end
-
-def generate_docs(type)
-  spec = spec_file(type)
-  cmd = type == :saas ? 'SAAS_SWAGGER=1' : 'SAAS_SWAGGER=0'
-  cmd << ' bundle exec source2swagger -f lib/3scale/backend/listener.rb -c "##~" -o docs/active_docs/.'
+def generate_docs
+  cmd = 'bundle exec source2swagger -f lib/3scale/backend/listener.rb -c "##~" -o docs/active_docs/.'
   run_command cmd
   # source2swagger is really bad at generating readable JSON
   require 'json'
-  File.write(spec, JSON.pretty_generate(JSON.parse(File.read(spec))) << "\n")
+  File.write(SPEC_FILE, JSON.pretty_generate(JSON.parse(File.read(SPEC_FILE))) << "\n")
 end
 
 namespace :docs do
   namespace :swagger do
-    namespace :generate do
-      # The swagger specs generated by these tasks need to be committed to the
-      # Porta repo: https://github.com/3scale/porta/tree/master/doc/active_docs
-      # (Service Management API (on-premises).json and Service Management
-      # API.json)
-
-      desc "Generates all swagger docs"
-      task all: [:saas, :on_prem]
-
-      desc "Generates swagger docs for SaaS"
-      task :saas do
-        generate_docs(:saas)
-      end
+    # The swagger specs generated by these tasks need to be committed to the
+    # Porta repo: https://github.com/3scale/porta/tree/master/doc/active_docs
+    # (Service Management API.json)
 
-      desc "Generates swagger docs for on-prem"
-      task :on_prem do
-        generate_docs(:on_prem)
-      end
+    desc "Generates swagger docs"
+    task :generate do
+      generate_docs
     end
   end
 end

data/licenses.xml

@@ -3,7 +3,7 @@
   <dependencies>
           <dependency>
         <packageName>activesupport</packageName>
-        <version>5.1.4</version>
+        <version>5.2.4.3</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>2.100.0</version>
+        <version>2.101.0</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -243,7 +243,7 @@
       </dependency>
           <dependency>
         <packageName>concurrent-ruby</packageName>
-        <version>1.0.5</version>
+        <version>1.1.6</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -318,16 +318,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>faraday</packageName>
-        <version>0.13.1</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>ffi</packageName>
@@ -338,16 +328,6 @@
               <url>http://opensource.org/licenses/BSD-3-Clause</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>geminabox</packageName>
-        <version>0.13.11</version>
-        <licenses>
-                      <license>
-              <name>MIT-LICENSE</name>
-              <url></url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>gli</packageName>
@@ -361,27 +341,17 @@
       </dependency>
           <dependency>
         <packageName>hiredis</packageName>
-        <version>0.6.1</version>
+        <version>0.6.3</version>
         <licenses>
                       <license>
               <name>New BSD</name>
               <url>http://opensource.org/licenses/BSD-3-Clause</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>httpclient</packageName>
-        <version>2.8.3</version>
-        <licenses>
-                      <license>
-              <name>ruby</name>
-              <url>http://www.ruby-lang.org/en/LICENSE.txt</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>i18n</packageName>
-        <version>0.9.1</version>
+        <version>1.8.2</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -471,7 +441,7 @@
       </dependency>
           <dependency>
         <packageName>minitest</packageName>
-        <version>5.10.3</version>
+        <version>5.14.1</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -512,16 +482,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>multipart-post</packageName>
-        <version>2.0.0</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>mustache</packageName>
@@ -542,16 +502,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>nesty</packageName>
-        <version>1.0.2</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>net-scp</packageName>
@@ -809,7 +759,7 @@
       </dependency>
           <dependency>
         <packageName>redis</packageName>
-        <version>4.1.1</version>
+        <version>4.1.3</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -826,16 +776,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>reentrant_flock</packageName>
-        <version>0.1.1</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>resque</packageName>
@@ -1123,7 +1063,7 @@
       </dependency>
           <dependency>
         <packageName>tzinfo</packageName>
-        <version>1.2.4</version>
+        <version>1.2.7</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 2.100.0
+  version: 2.101.0
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-22 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -88,7 +88,6 @@ files:
 - lib/3scale/backend/extensions/array.rb
 - lib/3scale/backend/extensions/hash.rb
 - lib/3scale/backend/extensions/nil_class.rb
-- lib/3scale/backend/extensions/redis.rb
 - lib/3scale/backend/extensions/string.rb
 - lib/3scale/backend/extensions/time.rb
 - lib/3scale/backend/failed_jobs_scheduler.rb

data/lib/3scale/backend/extensions/redis.rb

@@ -1,44 +0,0 @@
-# Monkey-patches a method in Redis::Client::Connector::Sentinel to fix a bug
-# with sentinel passwords. It applies the fix in
-# https://github.com/redis/redis-rb/pull/856.
-#
-# The fix was included in 4.1.2, but we cannot upgrade because that version
-# drops support for ruby < 2.3.0 which we still need to support.
-#
-# This should only be temporary. It should be deleted when updating the gem.
-class Redis
-  class Client
-    class Connector
-      class Sentinel
-        def sentinel_detect
-          @sentinels.each do |sentinel|
-            client = Redis::Client.new(@options.merge({:host => sentinel[:host],
-                                                       :port => sentinel[:port],
-                                                       password: sentinel[:password],
-                                                       :reconnect_attempts => 0,
-                                                      }))
-
-            begin
-              if result = yield(client)
-                # This sentinel responded. Make sure we ask it first next time.
-                @sentinels.delete(sentinel)
-                @sentinels.unshift(sentinel)
-
-                return result
-              end
-            rescue BaseConnectionError
-            rescue RuntimeError => exception
-              # Needed because when the sentinel address cannot be resolved it
-              # raises this instead of "BaseConnectionError"
-              raise unless exception.message =~ /Name or service not known/
-            ensure
-              client.disconnect
-            end
-          end
-
-          raise CannotConnectError, "No sentinels available."
-        end
-      end
-    end
-  end
-end