apisonator 2.100.2.pre1 → 3.0.0.pre1

data/lib/3scale/backend/metric/collection.rb

@@ -75,10 +75,7 @@ module ThreeScale
         end
 
         def load_metric_id(name)
-          Memoizer.memoize_block(Memoizer.build_key(self,
-                                        :load_metric_id, @service_id, name)) do
-            storage.get(encode_key("metric/service_id:#{@service_id}/name:#{name}/id"))
-          end || raise(MetricInvalid.new(name))
+          Metric.load_id(@service_id, name) || raise(MetricInvalid.new(name))
         end
 
         ## accepts postive integers or positive integers preffixed with # (for sets)

data/lib/3scale/backend/storage_async/client.rb

@@ -46,7 +46,9 @@ module ThreeScale
           port ||= DEFAULT_PORT
 
           endpoint = Async::IO::Endpoint.tcp(host, port)
-          @redis_async = Async::Redis::Client.new(endpoint)
+          @redis_async = Async::Redis::Client.new(
+            endpoint, connection_limit: opts[:max_connections]
+          )
           @building_pipeline = false
         end
 

data/lib/3scale/backend/storage_helpers.rb

@@ -52,13 +52,18 @@ module ThreeScale
               # duplicated commands because of this setting.
               reconnect_attempts: 1,
               # use by default the C extension client
-              driver: :hiredis
+              driver: :hiredis,
+              # applies only to async mode. The sync library opens 1 connection
+              # per process.
+              max_connections: 10,
           }.freeze
           private_constant :CONN_OPTIONS
 
           # CONN_WHITELIST - Connection options that can be specified in config
           # Note: we don't expose reconnect_attempts until the bug above is fixed
-          CONN_WHITELIST = [:connect_timeout, :read_timeout, :write_timeout].freeze
+          CONN_WHITELIST = [
+            :connect_timeout, :read_timeout, :write_timeout, :max_connections
+          ].freeze
           private_constant :CONN_WHITELIST
 
           # Parameters regarding target server we will take from a config object

data/lib/3scale/backend/transactor.rb

@@ -65,19 +65,7 @@ module ThreeScale
         params[:app_id] = nil if app_id && app_id.empty?
 
         if oauth
-          if app_id.nil?
-            access_token = params[:access_token]
-            access_token = nil if access_token && access_token.empty?
-
-            if access_token.nil?
-              raise ApplicationNotFound.new nil if app_id.nil?
-            else
-              app_id = get_token_ids(access_token, service_id, app_id)
-              # update params, since they are checked elsewhere
-              params[:app_id] = app_id
-            end
-          end
-
+          raise ApplicationNotFound.new nil if app_id.nil?
           validators = Validators::OAUTH_VALIDATORS
         else
           validators = Validators::VALIDATORS
@@ -87,9 +75,13 @@ module ThreeScale
         application = Application.load_by_id_or_user_key!(service_id,
                                                           app_id,
                                                           params[:user_key])
+
+        extensions = request_info && request_info[:extensions] || {}
+
+        preload_usage_limits(application, extensions[:no_body], params[:usage])
+
         now          = Time.now.getutc
         usage_values = Usage.application_usage(application, now)
-        extensions   = request_info && request_info[:extensions] || {}
         status_attrs = {
           service_id:      service_id,
           application:     application,
@@ -169,6 +161,27 @@ module ThreeScale
         Resque.enqueue(ReportJob, service_id, data, Time.now.getutc.to_f, context_info)
       end
 
+      # Loads the usage limits that are needed to authorize the current request.
+      # These are the cases:
+      # - When no_body is enabled, we only need to load the limits related with
+      # the metrics included in the request, that is, the ones included plus all
+      # their ancestors in the hierarchy. That's all we need to verify if the
+      # request is within the limits defined.
+      # - When no_body is disabled, we need to load all the limits because they
+      # are needed to generate the XML response.
+      # - When the usage reported in the request is empty, apisonator returns
+      # "limits_exceeded" if any of the limits defined has been violated. That's
+      # why in that scenario we need to load all the limits.
+      def preload_usage_limits(application, no_body_enabled, usage_in_params)
+        metric_names_in_usage = usage_in_params&.keys || {}
+
+        if no_body_enabled && !metric_names_in_usage.empty?
+          application.load_usage_limits_affected_by(metric_names_in_usage)
+        else
+          application.load_all_usage_limits
+        end
+      end
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/transactor/report_job.rb

@@ -59,7 +59,7 @@ module ThreeScale
             return [] if transactions.empty?
             transactions = transactions.values if transactions.respond_to?(:values)
             transactions.group_by do |transaction|
-              Application.extract_id!(service_id, transaction['app_id'], transaction['user_key'], transaction['access_token'])
+              Application.extract_id!(service_id, transaction['app_id'], transaction['user_key'])
             end.each(&block)
           end
 

data/lib/3scale/backend/transactor/status.rb

@@ -80,11 +80,6 @@ module ThreeScale
           values && values[usage_limit.metric_id] || 0
         end
 
-        def value_for_application_usage_limit(usage_limit)
-          values = @values[usage_limit.period]
-          values && values[usage_limit.metric_id] || 0
-        end
-
         # provides a hierarchy hash with metrics as symbolic names
         def hierarchy
           @hierarchy ||= Metric.hierarchy service_id

data/lib/3scale/backend/usage_limit.rb

@@ -22,20 +22,15 @@ module ThreeScale
 
         def load_all(service_id, plan_id)
           metric_ids = Metric.load_all_ids(service_id)
-          return metric_ids if metric_ids.empty?
-
-          results = []
-          with_pairs_and_values service_id, plan_id, metric_ids do |pair, value|
-            value and results << new(service_id: service_id,
-                                     plan_id: plan_id,
-                                     metric_id: pair[0],
-                                     period: pair[1],
-                                     value: value.to_i)
-          end
-          results
+          generate_for_metrics(service_id, plan_id, metric_ids)
         end
         memoize :load_all
 
+        def load_for_affecting_metrics(service_id, plan_id, metric_ids)
+          generate_for_metrics(service_id, plan_id, metric_ids)
+        end
+        memoize :load_for_affecting_metrics
+
         def load_value(service_id, plan_id, metric_id, period)
           raw_value = storage.get(key(service_id, plan_id, metric_id, period))
           raw_value and raw_value.to_i
@@ -80,6 +75,20 @@ module ThreeScale
           encode_key(key_pre + period.to_s)
         end
 
+        def generate_for_metrics(service_id, plan_id, metric_ids)
+          return metric_ids if metric_ids.empty?
+
+          results = []
+          with_pairs_and_values service_id, plan_id, metric_ids do |pair, value|
+            value and results << new(service_id: service_id,
+                                     plan_id: plan_id,
+                                     metric_id: pair[0],
+                                     period: pair[1],
+                                     value: value.to_i)
+          end
+          results
+        end
+
         # yields [pair(metric_id, period), value]
         def with_pairs_and_values(service_id, plan_id, metric_ids, &blk)
           pairs, values = get_pairs_and_values_for service_id, plan_id, metric_ids

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '2.100.2.pre1'
+    VERSION = '3.0.0.pre1'
   end
 end

data/licenses.xml

@@ -3,7 +3,7 @@
   <dependencies>
           <dependency>
         <packageName>activesupport</packageName>
-        <version>5.1.4</version>
+        <version>5.2.4.3</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>2.100.1</version>
+        <version>2.101.1</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -243,7 +243,7 @@
       </dependency>
           <dependency>
         <packageName>concurrent-ruby</packageName>
-        <version>1.0.5</version>
+        <version>1.1.6</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -318,16 +318,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>faraday</packageName>
-        <version>0.13.1</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>ffi</packageName>
@@ -338,16 +328,6 @@
               <url>http://opensource.org/licenses/BSD-3-Clause</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>geminabox</packageName>
-        <version>0.13.11</version>
-        <licenses>
-                      <license>
-              <name>MIT-LICENSE</name>
-              <url></url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>gli</packageName>
@@ -368,20 +348,10 @@
               <url>http://opensource.org/licenses/BSD-3-Clause</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>httpclient</packageName>
-        <version>2.8.3</version>
-        <licenses>
-                      <license>
-              <name>ruby</name>
-              <url>http://www.ruby-lang.org/en/LICENSE.txt</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>i18n</packageName>
-        <version>0.9.1</version>
+        <version>1.8.2</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -471,7 +441,7 @@
       </dependency>
           <dependency>
         <packageName>minitest</packageName>
-        <version>5.10.3</version>
+        <version>5.14.1</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -512,16 +482,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>multipart-post</packageName>
-        <version>2.0.0</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>mustache</packageName>
@@ -542,16 +502,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>nesty</packageName>
-        <version>1.0.2</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>net-scp</packageName>
@@ -826,16 +776,6 @@
               <url>http://opensource.org/licenses/mit-license</url>
             </license>
                   </licenses>
-      </dependency>
-          <dependency>
-        <packageName>reentrant_flock</packageName>
-        <version>0.1.1</version>
-        <licenses>
-                      <license>
-              <name>MIT</name>
-              <url>http://opensource.org/licenses/mit-license</url>
-            </license>
-                  </licenses>
       </dependency>
           <dependency>
         <packageName>resque</packageName>
@@ -1123,7 +1063,7 @@
       </dependency>
           <dependency>
         <packageName>tzinfo</packageName>
-        <version>1.2.4</version>
+        <version>1.2.7</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 2.100.2.pre1
+  version: 3.0.0.pre1
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-06-19 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -111,11 +111,6 @@ files:
 - lib/3scale/backend/memoizer.rb
 - lib/3scale/backend/metric.rb
 - lib/3scale/backend/metric/collection.rb
-- lib/3scale/backend/oauth.rb
-- lib/3scale/backend/oauth/token.rb
-- lib/3scale/backend/oauth/token_key.rb
-- lib/3scale/backend/oauth/token_storage.rb
-- lib/3scale/backend/oauth/token_value.rb
 - lib/3scale/backend/period.rb
 - lib/3scale/backend/period/boundary.rb
 - lib/3scale/backend/period/cache.rb
@@ -184,8 +179,6 @@ files:
 - lib/3scale/backend/validators/service_state.rb
 - lib/3scale/backend/validators/state.rb
 - lib/3scale/backend/version.rb
-- lib/3scale/backend/views/oauth_access_tokens.builder
-- lib/3scale/backend/views/oauth_app_id_by_token.builder
 - lib/3scale/backend/worker.rb
 - lib/3scale/backend/worker_async.rb
 - lib/3scale/backend/worker_metrics.rb

data/lib/3scale/backend/oauth.rb

@@ -1,4 +0,0 @@
-require '3scale/backend/oauth/token'
-require '3scale/backend/oauth/token_key'
-require '3scale/backend/oauth/token_value'
-require '3scale/backend/oauth/token_storage'

data/lib/3scale/backend/oauth/token.rb

@@ -1,26 +0,0 @@
-module ThreeScale
-  module Backend
-    module OAuth
-      class Token
-        attr_reader :service_id, :token, :key
-        attr_accessor :ttl, :app_id
-
-        def initialize(token, service_id, app_id, ttl)
-          @token = token
-          @service_id = service_id
-          @app_id = app_id
-          @ttl = ttl
-          @key = Key.for token, service_id
-        end
-
-        def value
-          Value.for app_id
-        end
-
-        def self.from_value(token, service_id, value, ttl)
-          new token, service_id, *Value.from(value), ttl
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/oauth/token_key.rb

@@ -1,30 +0,0 @@
-# This module defines the format of the keys for OAuth tokens and token sets.
-#
-# Note that while we can build the key easily, we cannot reliably obtain a token
-# and a service_id out of the key, because there are no constraints on them:
-#
-# "oauth_access_tokens/service:some/servicegoeshere/andthisis_a_/valid_token"
-#
-module ThreeScale
-  module Backend
-    module OAuth
-      class Token
-        module Key
-          class << self
-            def for(token, service_id)
-              "oauth_access_tokens/service:#{service_id}/#{token}"
-            end
-          end
-
-          module Set
-            class << self
-              def for(service_id, app_id)
-                "oauth_access_tokens/service:#{service_id}/app:#{app_id}/"
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/oauth/token_storage.rb

@@ -1,313 +0,0 @@
-module ThreeScale
-  module Backend
-    module OAuth
-      class Token
-        module Storage
-          include Configurable
-
-          # Default token size is 4K - 512 (to allow for some metadata)
-          MAXIMUM_TOKEN_SIZE = configuration.oauth.max_token_size || 3584
-          private_constant :MAXIMUM_TOKEN_SIZE
-          TOKEN_MAX_REDIS_SLICE_SIZE = 500
-          private_constant :TOKEN_MAX_REDIS_SLICE_SIZE
-          TOKEN_TTL_DEFAULT = 86400
-          private_constant :TOKEN_TTL_DEFAULT
-          TOKEN_TTL_PERMANENT = 0
-          private_constant :TOKEN_TTL_PERMANENT
-
-          Error = Class.new StandardError
-          InconsistencyError = Class.new Error
-
-          class << self
-            include Backend::Logging
-            include Backend::StorageHelpers
-
-            def create(token, service_id, app_id, ttl = nil)
-              raise AccessTokenFormatInvalid if token.nil? || token.empty? ||
-                !token.is_a?(String) || token.bytesize > MAXIMUM_TOKEN_SIZE
-
-              # raises if TTL is invalid
-              ttl = sanitized_ttl ttl
-
-              key = Key.for token, service_id
-              raise AccessTokenAlreadyExists.new(token) unless storage.get(key).nil?
-
-              value = Value.for app_id
-              token_set = Key::Set.for(service_id, app_id)
-
-              store_token token, token_set, key, value, ttl
-              ensure_stored! token, token_set, key, value
-            end
-
-            # Deletes a token
-            #
-            # Returns the associated app_id or nil
-            #
-            def delete(token, service_id)
-              key = Key.for token, service_id
-              val = storage.get key
-              if val
-                app_id = Value.from val
-                token_set = Key::Set.for(service_id, app_id)
-
-                existed, * = remove_a_token token_set, token, key
-
-                unless existed
-                  logger.notify(InconsistencyError.new("Found OAuth token " \
-                    "#{token} for service #{service_id} and app #{app_id} as " \
-                    "key but not in set!"))
-                end
-              end
-
-              val
-            end
-
-            # Get a token's associated app_id
-            def get_credentials(token, service_id)
-              app_id = Value.from(storage.get(Key.for(token, service_id)))
-              raise AccessTokenInvalid.new token if app_id.nil?
-              app_id
-            end
-
-            # This is used to list tokens by service and app.
-            #
-            # Note: this deletes tokens that have not been found from the set of
-            # tokens for the given app - those have to be expired tokens.
-            def all_by_service_and_app(service_id, app_id)
-              token_set = Key::Set.for(service_id, app_id)
-              deltokens = []
-              tokens_n_values_flat(token_set, service_id)
-                .select do |(token, _key, value, _ttl)|
-                  app_id = Value.from value
-                  if app_id.nil?
-                    deltokens << token
-                    false
-                  else
-                    true
-                  end
-                end
-                .map do |(token, _key, value, ttl)|
-                  Token.from_value token, service_id, value, ttl
-                end
-                .tap do
-                  # delete expired tokens (nil values) from token set
-                  deltokens.each_slice(TOKEN_MAX_REDIS_SLICE_SIZE) do |delgrp|
-                    storage.srem token_set, delgrp
-                  end
-                end
-            end
-
-            # Remove tokens by app_id.
-            #
-            # Triggered by Application deletion.
-            #
-            def remove_tokens(service_id, app_id)
-              remove_tokens_by service_id, app_id
-            end
-
-
-            private
-
-            # Remove all tokens
-            #
-            # I thought of leaving this one public, but remove_*_tokens removed
-            # my use cases for the time being.
-            def remove_tokens_by(service_id, app_id)
-              token_set = Key::Set.for(service_id, app_id)
-
-              remove_whole_token_set(token_set, service_id)
-            end
-
-            def remove_token_set_by(token_set, service_id, &blk)
-              # Get tokens. Filter them. Group them into manageable groups.
-              # Extract tokens and keys into separate arrays, one for each.
-              # Remove tokens from token set (they are keys in a set) and token
-              # keys themselves.
-              tokens_n_values_flat(token_set, service_id, false)
-              .select(&blk)
-              .each_slice(TOKEN_MAX_REDIS_SLICE_SIZE)
-              .inject([[], []]) do |acc, groups|
-                groups.each do |token, key, _value|
-                  acc[0] << token
-                  acc[1] << key
-                end
-                acc
-              end
-              .each_slice(2)
-              .inject([]) do |acc, (tokens, keys)|
-                storage.pipelined do
-                  if tokens && !tokens.empty?
-                    storage.srem token_set, tokens
-                    acc.concat tokens
-                  end
-                  storage.del keys if keys && !keys.empty?
-                end
-                acc
-              end
-            end
-
-            def remove_a_token(token_set, token, key)
-              storage.pipelined do
-                storage.srem token_set, token
-                storage.del key
-              end
-            end
-
-            def remove_whole_token_set(token_set, service_id)
-              _token_groups, key_groups = tokens_n_keys(token_set, service_id)
-              storage.pipelined do
-                storage.del token_set
-                # remove all tokens for this app
-                key_groups.each do |keys|
-                  storage.del keys
-                end
-              end
-            end
-
-            # TODO: provide a SSCAN interface with lazy enums because SMEMBERS
-            # is prone to DoSing and timeouts
-            def tokens_from(token_set)
-              storage.smembers(token_set)
-            end
-
-            def tokens_n_keys(token_set, service_id)
-              token_groups = tokens_from(token_set).each_slice(TOKEN_MAX_REDIS_SLICE_SIZE)
-              key_groups = token_groups.map do |tokens|
-                tokens.map do |token|
-                  Key.for token, service_id
-                end
-              end
-
-              [token_groups, key_groups]
-            end
-
-            # Provides grouped data which matches respectively in each array
-            # position, ie. 1st group of data contains a group of tokens, keys
-            # and values with ttls, and position N of the tokens group has key
-            # in position N of the keys group, and so on.
-            #
-            # [[[token group], [key group], [value_with_ttls_group]], ...]
-            #
-            def tokens_n_values_groups(token_set, service_id, with_ttls)
-              token_groups, key_groups = tokens_n_keys(token_set, service_id)
-              value_ttl_groups = key_groups.map do |keys|
-                # pipelining will create an array with the results of commands
-                res = storage.pipelined do
-                  storage.mget(keys)
-                  if with_ttls
-                    keys.map do |key|
-                      storage.ttl key
-                    end
-                  end
-                end
-                # [mget array, 0..n ttls] => [mget array, ttls array]
-                [res.shift, res]
-              end
-              token_groups.zip(key_groups, value_ttl_groups)
-            end
-
-            # Zips the data provided by tokens_n_values_groups so that you stop
-            # looking at indexes in the respective arrays and instead have:
-            #
-            # [group 0, ..., group N] where each group is made of:
-            #   [[token 0, key 0, value 0, ttl 0], ..., [token N, key N, value
-            #   N, ttl N]]
-            #
-            def tokens_n_values_zipped_groups(token_set, service_id, with_ttls = true)
-              tokens_n_values_groups(token_set,
-                                     service_id,
-                                     with_ttls).map do |tokens, keys, (values, ttls)|
-                tokens.zip keys, values, ttls
-              end
-            end
-
-            # Flattens the data provided by tokens_n_values_zipped_groups so
-            # that you have a transparent iterator with all needed data and can
-            # stop worrying about streaming groups of elements.
-            #
-            def tokens_n_values_flat(token_set, service_id, with_ttls = true)
-              tokens_n_values_zipped_groups(token_set,
-                                            service_id,
-                                            with_ttls).flat_map do |groups|
-                groups.map do |token, key, value, ttl|
-                  [token, key, value, ttl]
-                end
-              end
-            end
-
-            # Store the specified token in Redis
-            #
-            # TTL specified in seconds.
-            # A TTL of 0 stores a permanent token
-            def store_token(token, token_set, key, value, ttl)
-              # build the storage command so that we can pipeline everything cleanly
-              command = :set
-              args = [key]
-
-              if !permanent_ttl? ttl
-                command = :setex
-                args << ttl
-              end
-
-              args << value
-
-              # pipelined will return nil if it is embedded into another
-              # pipeline(which would be an error at this point) or if shutting
-              # down and a connection error happens. Both things being abnormal
-              # means we should just raise a storage error.
-              raise AccessTokenStorageError, token unless storage.pipelined do
-                storage.send(command, *args)
-                storage.sadd(token_set, token)
-              end
-            end
-
-
-            # Make sure everything ended up there
-            #
-            # TODO: review and possibly reimplement trying to leave it
-            # consistent as much as possible.
-            #
-            # Note that we have a sharding proxy and pipelines can't be guaranteed
-            # to behave like transactions, since we might have one non-working
-            # shard. Instead of relying on proxy-specific responses, we just check
-            # that the data we should have in the store is really there.
-            def ensure_stored!(token, token_set, key, value)
-              results = storage.pipelined do
-                storage.get(key)
-                storage.sismember(token_set, token)
-              end
-
-              results.last && results.first == value ||
-                raise(AccessTokenStorageError, token)
-            end
-
-	    # Validation for the TTL value
-	    #
-	    # 0 is accepted (understood as permanent token)
-	    # Negative values are not accepted
-	    # Integer(ttl) validation is required (if input is nil, default applies)
-	    def sanitized_ttl(ttl)
-              ttl = begin
-                      Integer(ttl)
-                    rescue TypeError
-                      # ttl is nil
-                      TOKEN_TTL_DEFAULT
-                    rescue
-                      # NaN
-                      -1
-                    end
-              raise AccessTokenInvalidTTL if ttl < 0
-
-	      ttl
-            end
-
-	    # Check whether a TTL has the magic value for a permanent token
-	    def permanent_ttl?(ttl)
-              ttl == TOKEN_TTL_PERMANENT
-	    end
-          end
-        end
-      end
-    end
-  end
-end