apisonator 2.100.2 → 2.101.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36a24cd40b3b9d707a5b4c06243a17c11fa0d3e42929dd0c5f5edc5aae5e35fd
-  data.tar.gz: c523234e3ce2afeaf1b536a99b138532020643574cd24173e8457187c1cc9693
+  metadata.gz: 4740b1613c8bd2182adf9587db2a2ae6667a50ee140c99c679c232b6da3b7193
+  data.tar.gz: 0e34376e2e788194b44eeb5aedf7d8654110c8baa4a0885ed2c55a3a28fcb0c1
 SHA512:
-  metadata.gz: 93da0370affbadcbf04cba8e213faa5d7cc2ff03ec2e848436596c54b5fec05d6bf0847da4d799243d62c46f7cab6c8121010c975a174b6d5beb5f4d63280019
-  data.tar.gz: '07685172c1e7f102a9fe0ed967cbcd61dfbbe138d8f6447a0183e5a6ba28a7873b96e52ff72206d268fa233642f1195de0b6a8a7ab777e4e90f39146d35fa618'
+  metadata.gz: c6431c06f07433710308ef8b9121d2975415cbec27d2c82db3c8f13c4a0dab364b6a9085a550a530bd000a18234bfd63d67ee555ea69299f8745734725f9f436
+  data.tar.gz: 90618d349784bfe5ff9d78b940f93d7b63013fb9f1ec1b9c68f03881ff9e13a165d0532a98d50c26218402cc197c3f9f730f7b34ca26589cb8f4b91fa37e6cab

data/CHANGELOG.md

@@ -2,6 +2,24 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 2.101.0 - 2020-06-04
+
+### Added
+
+- Introduced the `CONFIG_REDIS_MAX_CONNS` and `CONFIG_QUEUES_MAX_CONNS` ENVs to
+configure the max number of Redis connections when using the async mode
+([#214](https://github.com/3scale/apisonator/pull/214)).
+
+### Changed
+
+- Perf optimization: loading the usage limits is now done more efficiently.
+There is a noticeable improvement in requests with `no_body` enabled for
+services with many metrics defined
+([#221](https://github.com/3scale/apisonator/pull/221)).
+- Updated activesupport to 5.2.4.3
+([#217](https://github.com/3scale/apisonator/pull/217)).
+
+
 ## 2.100.2 - 2020-05-08
 
 ### Changed

data/Gemfile.base

@@ -25,7 +25,6 @@ group :test do
   gem 'resque_spec',   '~> 0.17.0'
   gem 'timecop',       '~> 0.9.1'
   gem 'rspec',         '~> 3.7.0', require: nil
-  gem 'geminabox',     '~> 0.13.11', require: false
   gem 'codeclimate-test-reporter', '~> 0.6.0', require: nil
   gem 'async-rspec'
 end

data/Gemfile.lock

@@ -35,14 +35,14 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (2.100.2)
+    apisonator (2.101.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.2.4.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     airbrake (4.3.1)
@@ -95,7 +95,7 @@ GEM
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
     coderay (1.1.2)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
     diff-lcs (1.3)
@@ -112,20 +112,10 @@ GEM
       process-metrics (~> 0.1.0)
       rack (>= 1.0)
       samovar (~> 2.1)
-    faraday (0.13.1)
-      multipart-post (>= 1.2, < 3)
     ffi (1.12.2)
-    geminabox (0.13.11)
-      builder
-      faraday
-      httpclient (>= 2.2.7)
-      nesty
-      reentrant_flock
-      sinatra (>= 1.2.7)
     gli (2.16.1)
     hiredis (0.6.3)
-    httpclient (2.8.3)
-    i18n (0.9.1)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jmespath (1.3.1)
     json (2.1.0)
@@ -141,15 +131,13 @@ GEM
     metaclass (0.0.4)
     method_source (0.9.0)
     mini_portile2 (2.4.0)
-    minitest (5.10.3)
+    minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
     multi_json (1.13.1)
-    multipart-post (2.0.0)
     mustache (1.0.5)
     mustermann (1.0.2)
-    nesty (1.0.2)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
@@ -191,7 +179,6 @@ GEM
     rake (13.0.1)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
-    reentrant_flock (0.1.1)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
       rspec-core (>= 3.0.0)
@@ -256,7 +243,7 @@ GEM
     timers (4.3.0)
     toml (0.2.0)
       parslet (~> 1.8.0)
-    tzinfo (1.2.4)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     vegas (0.1.11)
       rack (>= 1.0.0)
@@ -288,7 +275,6 @@ DEPENDENCIES
   codeclimate-test-reporter (~> 0.6.0)
   daemons (= 1.2.4)
   falcon (~> 0.35)
-  geminabox (~> 0.13.11)
   gli (~> 2.16.1)
   hiredis (~> 0.6.1)
   license_finder (~> 5)

data/Gemfile.on_prem.lock

@@ -35,14 +35,14 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (2.100.2)
+    apisonator (2.101.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.1.4)
+    activesupport (5.2.4.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     async (1.24.2)
@@ -85,7 +85,7 @@ GEM
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
     coderay (1.1.2)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.6)
     console (1.8.2)
     daemons (1.2.4)
     diff-lcs (1.3)
@@ -102,20 +102,10 @@ GEM
       process-metrics (~> 0.1.0)
       rack (>= 1.0)
       samovar (~> 2.1)
-    faraday (0.13.1)
-      multipart-post (>= 1.2, < 3)
     ffi (1.12.2)
-    geminabox (0.13.11)
-      builder
-      faraday
-      httpclient (>= 2.2.7)
-      nesty
-      reentrant_flock
-      sinatra (>= 1.2.7)
     gli (2.16.1)
     hiredis (0.6.3)
-    httpclient (2.8.3)
-    i18n (0.9.1)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     json (2.1.0)
     license_finder (5.9.2)
@@ -130,15 +120,13 @@ GEM
     metaclass (0.0.4)
     method_source (0.9.0)
     mini_portile2 (2.4.0)
-    minitest (5.10.3)
+    minitest (5.14.1)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
     mono_logger (1.1.0)
     multi_json (1.13.1)
-    multipart-post (2.0.0)
     mustache (1.0.5)
     mustermann (1.0.2)
-    nesty (1.0.2)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
@@ -179,7 +167,6 @@ GEM
     rake (13.0.1)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
-    reentrant_flock (0.1.1)
     resque_spec (0.17.0)
       resque (>= 1.19.0)
       rspec-core (>= 3.0.0)
@@ -242,7 +229,7 @@ GEM
     timers (4.3.0)
     toml (0.2.0)
       parslet (~> 1.8.0)
-    tzinfo (1.2.4)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     vegas (0.1.11)
       rack (>= 1.0.0)
@@ -270,7 +257,6 @@ DEPENDENCIES
   codeclimate-test-reporter (~> 0.6.0)
   daemons (= 1.2.4)
   falcon (~> 0.35)
-  geminabox (~> 0.13.11)
   gli (~> 2.16.1)
   hiredis (~> 0.6.1)
   license_finder (~> 5)

data/lib/3scale/backend/application.rb

@@ -223,6 +223,24 @@ module ThreeScale
         @usage_limits ||= UsageLimit.load_all(service_id, plan_id)
       end
 
+      def load_all_usage_limits
+        @usage_limits = UsageLimit.load_all(service_id, plan_id)
+      end
+
+      # Loads the usage limits affected by the metrics received, that is, the
+      # limits that are defined for those metrics plus all their ancestors in
+      # the metrics hierarchy.
+      def load_usage_limits_affected_by(metric_names)
+        metric_ids = metric_names.flat_map do |name|
+          [name] + Metric.ascendants(service_id, name)
+        end.uniq.map do |name|
+          Metric.load_id(service_id, name)
+        end
+
+        # IDs are sorted to be able to use the memoizer
+        @usage_limits = UsageLimit.load_for_affecting_metrics(service_id, plan_id, metric_ids.sort)
+      end
+
       def active?
         state == :active
       end

data/lib/3scale/backend/configuration.rb

@@ -46,9 +46,9 @@ module ThreeScale
 
       # Add configuration sections
       config.add_section(:queues, :master_name, :sentinels, :role,
-                         :connect_timeout, :read_timeout, :write_timeout)
+                         :connect_timeout, :read_timeout, :write_timeout, :max_connections)
       config.add_section(:redis, :url, :proxy, :sentinels, :role,
-                         :connect_timeout, :read_timeout, :write_timeout,
+                         :connect_timeout, :read_timeout, :write_timeout, :max_connections,
                          :async)
       config.add_section(:analytics_redis, :server,
                          :connect_timeout, :read_timeout, :write_timeout)

data/lib/3scale/backend/listener.rb

@@ -604,10 +604,6 @@ module ThreeScale
         end
       end
 
-      def application
-        @application ||= Application.load_by_id_or_user_key!(service_id, params[:app_id], params[:user_key])
-      end
-
       def service_id
         if params[:service_id].nil? || params[:service_id].empty?
           @service_id ||= Service.default_id!(params[:provider_key])

data/lib/3scale/backend/metric/collection.rb

@@ -75,10 +75,7 @@ module ThreeScale
         end
 
         def load_metric_id(name)
-          Memoizer.memoize_block(Memoizer.build_key(self,
-                                        :load_metric_id, @service_id, name)) do
-            storage.get(encode_key("metric/service_id:#{@service_id}/name:#{name}/id"))
-          end || raise(MetricInvalid.new(name))
+          Metric.load_id(@service_id, name) || raise(MetricInvalid.new(name))
         end
 
         ## accepts postive integers or positive integers preffixed with # (for sets)

data/lib/3scale/backend/storage_async/client.rb

@@ -46,7 +46,9 @@ module ThreeScale
           port ||= DEFAULT_PORT
 
           endpoint = Async::IO::Endpoint.tcp(host, port)
-          @redis_async = Async::Redis::Client.new(endpoint)
+          @redis_async = Async::Redis::Client.new(
+            endpoint, connection_limit: opts[:max_connections]
+          )
           @building_pipeline = false
         end
 

data/lib/3scale/backend/storage_helpers.rb

@@ -52,13 +52,18 @@ module ThreeScale
               # duplicated commands because of this setting.
               reconnect_attempts: 1,
               # use by default the C extension client
-              driver: :hiredis
+              driver: :hiredis,
+              # applies only to async mode. The sync library opens 1 connection
+              # per process.
+              max_connections: 10,
           }.freeze
           private_constant :CONN_OPTIONS
 
           # CONN_WHITELIST - Connection options that can be specified in config
           # Note: we don't expose reconnect_attempts until the bug above is fixed
-          CONN_WHITELIST = [:connect_timeout, :read_timeout, :write_timeout].freeze
+          CONN_WHITELIST = [
+            :connect_timeout, :read_timeout, :write_timeout, :max_connections
+          ].freeze
           private_constant :CONN_WHITELIST
 
           # Parameters regarding target server we will take from a config object

data/lib/3scale/backend/transactor.rb

@@ -87,9 +87,13 @@ module ThreeScale
         application = Application.load_by_id_or_user_key!(service_id,
                                                           app_id,
                                                           params[:user_key])
+
+        extensions = request_info && request_info[:extensions] || {}
+
+        preload_usage_limits(application, extensions[:no_body], params[:usage])
+
         now          = Time.now.getutc
         usage_values = Usage.application_usage(application, now)
-        extensions   = request_info && request_info[:extensions] || {}
         status_attrs = {
           service_id:      service_id,
           application:     application,
@@ -169,6 +173,27 @@ module ThreeScale
         Resque.enqueue(ReportJob, service_id, data, Time.now.getutc.to_f, context_info)
       end
 
+      # Loads the usage limits that are needed to authorize the current request.
+      # These are the cases:
+      # - When no_body is enabled, we only need to load the limits related with
+      # the metrics included in the request, that is, the ones included plus all
+      # their ancestors in the hierarchy. That's all we need to verify if the
+      # request is within the limits defined.
+      # - When no_body is disabled, we need to load all the limits because they
+      # are needed to generate the XML response.
+      # - When the usage reported in the request is empty, apisonator returns
+      # "limits_exceeded" if any of the limits defined has been violated. That's
+      # why in that scenario we need to load all the limits.
+      def preload_usage_limits(application, no_body_enabled, usage_in_params)
+        metric_names_in_usage = usage_in_params&.keys || {}
+
+        if no_body_enabled && !metric_names_in_usage.empty?
+          application.load_usage_limits_affected_by(metric_names_in_usage)
+        else
+          application.load_all_usage_limits
+        end
+      end
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/transactor/status.rb

@@ -80,11 +80,6 @@ module ThreeScale
           values && values[usage_limit.metric_id] || 0
         end
 
-        def value_for_application_usage_limit(usage_limit)
-          values = @values[usage_limit.period]
-          values && values[usage_limit.metric_id] || 0
-        end
-
         # provides a hierarchy hash with metrics as symbolic names
         def hierarchy
           @hierarchy ||= Metric.hierarchy service_id

data/lib/3scale/backend/usage_limit.rb

@@ -22,20 +22,15 @@ module ThreeScale
 
         def load_all(service_id, plan_id)
           metric_ids = Metric.load_all_ids(service_id)
-          return metric_ids if metric_ids.empty?
-
-          results = []
-          with_pairs_and_values service_id, plan_id, metric_ids do |pair, value|
-            value and results << new(service_id: service_id,
-                                     plan_id: plan_id,
-                                     metric_id: pair[0],
-                                     period: pair[1],
-                                     value: value.to_i)
-          end
-          results
+          generate_for_metrics(service_id, plan_id, metric_ids)
         end
         memoize :load_all
 
+        def load_for_affecting_metrics(service_id, plan_id, metric_ids)
+          generate_for_metrics(service_id, plan_id, metric_ids)
+        end
+        memoize :load_for_affecting_metrics
+
         def load_value(service_id, plan_id, metric_id, period)
           raw_value = storage.get(key(service_id, plan_id, metric_id, period))
           raw_value and raw_value.to_i
@@ -80,6 +75,20 @@ module ThreeScale
           encode_key(key_pre + period.to_s)
         end
 
+        def generate_for_metrics(service_id, plan_id, metric_ids)
+          return metric_ids if metric_ids.empty?
+
+          results = []
+          with_pairs_and_values service_id, plan_id, metric_ids do |pair, value|
+            value and results << new(service_id: service_id,
+                                     plan_id: plan_id,
+                                     metric_id: pair[0],
+                                     period: pair[1],
+                                     value: value.to_i)
+          end
+          results
+        end
+
         # yields [pair(metric_id, period), value]
         def with_pairs_and_values(service_id, plan_id, metric_ids, &blk)
           pairs, values = get_pairs_and_values_for service_id, plan_id, metric_ids

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '2.100.2'
+    VERSION = '2.101.0'
   end
 end

data/licenses.xml

@@ -3,7 +3,7 @@
   <dependencies>
           <dependency>
         <packageName>activesupport</packageName>
-        <version>5.1.4</version>
+        <version>5.2.4.3</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>2.100.2</version>
+        <version>2.101.0</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -243,7 +243,7 @@
       </dependency>
           <dependency>
         <packageName>concurrent-ruby</packageName>
-        <version>1.0.5</version>
+        <version>1.1.6</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -318,16 +318,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>faraday</packageName>
-        <version>0.13.1</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>ffi</packageName>
@@ -338,16 +328,6 @@
               <url>http://opensource.org/licenses/BSD-3-Clause</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>geminabox</packageName>
-        <version>0.13.11</version>
-        <licenses>
-                      <license>
-              <name>MIT-LICENSE</name>
-              <url></url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>gli</packageName>
@@ -368,20 +348,10 @@
               <url>http://opensource.org/licenses/BSD-3-Clause</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>httpclient</packageName>
-        <version>2.8.3</version>
-        <licenses>
-                      <license>
-              <name>ruby</name>
-              <url>http://www.ruby-lang.org/en/LICENSE.txt</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>i18n</packageName>
-        <version>0.9.1</version>
+        <version>1.8.2</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -471,7 +441,7 @@
       </dependency>
           <dependency>
         <packageName>minitest</packageName>
-        <version>5.10.3</version>
+        <version>5.14.1</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -512,16 +482,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>multipart-post</packageName>
-        <version>2.0.0</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>mustache</packageName>
@@ -542,16 +502,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>nesty</packageName>
-        <version>1.0.2</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>net-scp</packageName>
@@ -826,16 +776,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>reentrant_flock</packageName>
-        <version>0.1.1</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>resque</packageName>
@@ -1123,7 +1063,7 @@
       </dependency>
           <dependency>
         <packageName>tzinfo</packageName>
-        <version>1.2.4</version>
+        <version>1.2.7</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 2.100.2
+  version: 2.101.0
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.