apisonator 2.101.1 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17eddc012f9777873f8bb1eca063c269903980531ae2963e839c265e2cce8ecc
-  data.tar.gz: f0b5ff06a98132e51dd5ab4bbbcf9322f037ee029487919366fca1eb207c13e8
+  metadata.gz: edd1887f6aa71c17fb5279f98988c9a3b5e631e80bc1f4142b6619b5f5e24f48
+  data.tar.gz: b6478d14e88a177480396a7d46d5f4e33cea683aea17303175ad18ae646e546f
 SHA512:
-  metadata.gz: 81a05e8249f1a354897379fc814a898f712ecc9b5a82e6dd9a28ab21abbdb6bd2e7af65a71cb77571cf476d581a251773a87bc3b489bf5be2d9740d6e3e1812b
-  data.tar.gz: fe7bd84d54a37680841df80cd405f2b71ff4a1cb936d757df5aa06e484c7c871584101cac6ea3f72eb1b17836b3af188474590a9ab6d40c4919037a8ce1a644e
+  metadata.gz: 944b75385817abc4d2f8828c61b2301d5cdfbb6c1bb681b7a3165590f23ad1b6c80451fcf579b8f3cf0010fe5a95e65605998f5d8ec94f1ff4ce690556b3011a
+  data.tar.gz: f3f44b1b2ebfe6c8fcbbf36b90dd44e5a7875d997a36b97288417472fa4156689508bfc95afdf196a7a168f768aa6e38521502b93078f128994959d184c1cd37

data/CHANGELOG.md

@@ -2,6 +2,56 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.1.0 - 2020-10-14
+
+### Added
+
+- Prometheus metrics for the internal API
+([#236](https://github.com/3scale/apisonator/pull/236)).
+- Docs with a detailed explanation about how counter updates are performed
+([#239](https://github.com/3scale/apisonator/pull/239)).
+
+### Changed
+
+- NotifyJobs are run only when the service ID is explicitly defined
+([#238](https://github.com/3scale/apisonator/pull/238)).
+
+### Fixed
+
+- Fixed corner case that raised "TransactionTimestampNotWithinRange" in notify
+jobs ([#235](https://github.com/3scale/apisonator/pull/235)).
+
+
+## 3.0.1.1 - 2020-07-28
+
+### Changed
+
+- Updated json gem to v2.3.1
+([#232](https://github.com/3scale/apisonator/pull/232)).
+
+## 3.0.1 - 2020-07-14
+
+### Fixed
+
+- The Prometheus counter of "5xx" errors has been fixed
+([#230](https://github.com/3scale/apisonator/pull/230)).
+
+### Changed
+
+- Gem updates: rack to v2.1.4
+([#228](https://github.com/3scale/apisonator/pull/228)) and async-redis to
+v0.5.0 ([#229](https://github.com/3scale/apisonator/pull/229)).
+
+
+## 3.0.0 - 2020-06-19
+
+### Removed
+
+- Apisonator no longer supports the native oauth authorization mode. This is a
+feature that was deprecated a long time ago
+([#226](https://github.com/3scale/apisonator/pull/226)).
+
+
 ## 2.101.1 - 2020-06-05
 
 ### Fixed

data/Gemfile.base

@@ -53,13 +53,13 @@ gem 'rake', '~> 13.0'
 gem 'builder', '= 3.2.3'
 # Use a patched resque to allow reusing their Airbrake Failure class
 gem 'resque', git: 'https://github.com/3scale/resque', branch: '3scale'
-gem 'rack', '~> 2.0.8'
+gem 'rack', '~> 2.1.4'
 gem 'sinatra', '~> 2.0.3'
 gem 'sinatra-contrib', '~> 2.0.3'
 # Optional external error logging services
 gem 'bugsnag', '~> 6', require: nil
 gem 'yabeda-prometheus', '~> 0.5.0'
-gem 'async-redis', '~> 0.4.1'
+gem 'async-redis', '~> 0.5'
 gem 'falcon', '~> 0.35'
 
 # Use a patched redis-rb that fixes an issue when trying to connect with

data/Gemfile.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (2.101.1)
+    apisonator (3.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -66,14 +66,15 @@ GEM
     async-http-cache (0.1.5)
       async-http
       protocol-http (~> 0.14)
-    async-io (1.27.5)
+    async-io (1.27.7)
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.4.1)
+    async-redis (0.5.0)
       async (~> 1.8)
       async-io (~> 1.10)
-      protocol-redis (~> 0.2.0)
+      async-pool (~> 0.2)
+      protocol-redis (~> 0.5.0)
     async-rspec (1.13.0)
       rspec (~> 3.0)
       rspec-files (~> 1.0)
@@ -118,7 +119,7 @@ GEM
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jmespath (1.3.1)
-    json (2.1.0)
+    json (2.3.1)
     license_finder (5.9.2)
       bundler
       rubyzip
@@ -161,7 +162,7 @@ GEM
     protocol-http2 (0.11.6)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
-    protocol-redis (0.2.0)
+    protocol-redis (0.5.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -171,7 +172,7 @@ GEM
     pry-doc (0.11.1)
       pry (~> 0.9)
       yard (~> 0.9)
-    rack (2.0.9)
+    rack (2.1.4)
     rack-protection (2.0.3)
       rack
     rack-test (0.8.2)
@@ -266,7 +267,7 @@ PLATFORMS
 DEPENDENCIES
   airbrake (= 4.3.1)
   apisonator!
-  async-redis (~> 0.4.1)
+  async-redis (~> 0.5)
   async-rspec
   aws-sdk (= 2.4.2)
   benchmark-ips (~> 2.7.2)
@@ -286,7 +287,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   pry-doc (~> 0.11.1)
   puma!
-  rack (~> 2.0.8)
+  rack (~> 2.1.4)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
   redis!

data/Gemfile.on_prem.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (2.101.1)
+    apisonator (3.1.0)
 
 GEM
   remote: https://rubygems.org/
@@ -63,14 +63,15 @@ GEM
     async-http-cache (0.1.5)
       async-http
       protocol-http (~> 0.14)
-    async-io (1.27.5)
+    async-io (1.27.7)
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.4.1)
+    async-redis (0.5.0)
       async (~> 1.8)
       async-io (~> 1.10)
-      protocol-redis (~> 0.2.0)
+      async-pool (~> 0.2)
+      protocol-redis (~> 0.5.0)
     async-rspec (1.13.0)
       rspec (~> 3.0)
       rspec-files (~> 1.0)
@@ -107,7 +108,7 @@ GEM
     hiredis (0.6.3)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
+    json (2.3.1)
     license_finder (5.9.2)
       bundler
       rubyzip
@@ -149,7 +150,7 @@ GEM
     protocol-http2 (0.11.6)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
-    protocol-redis (0.2.0)
+    protocol-redis (0.5.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -159,7 +160,7 @@ GEM
     pry-doc (0.11.1)
       pry (~> 0.9)
       yard (~> 0.9)
-    rack (2.0.9)
+    rack (2.1.4)
     rack-protection (2.0.3)
       rack
     rack-test (0.8.2)
@@ -249,7 +250,7 @@ PLATFORMS
 
 DEPENDENCIES
   apisonator!
-  async-redis (~> 0.4.1)
+  async-redis (~> 0.5)
   async-rspec
   benchmark-ips (~> 2.7.2)
   bugsnag (~> 6)
@@ -267,7 +268,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   pry-doc (~> 0.11.1)
   puma!
-  rack (~> 2.0.8)
+  rack (~> 2.1.4)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
   redis!

data/lib/3scale/backend.rb

@@ -35,7 +35,6 @@ require '3scale/backend/rack'
 require '3scale/backend/extensions'
 require '3scale/backend/background_job'
 require '3scale/backend/storage'
-require '3scale/backend/oauth'
 require '3scale/backend/memoizer'
 require '3scale/backend/application'
 require '3scale/backend/error_storage'

data/lib/3scale/backend/application.rb

@@ -91,8 +91,8 @@ module ThreeScale
           end
         end
 
-        def extract_id!(service_id, app_id, user_key, access_token)
-          with_app_id_from_params service_id, app_id, user_key, access_token do |appid|
+        def extract_id!(service_id, app_id, user_key)
+          with_app_id_from_params service_id, app_id, user_key do |appid|
             exists? service_id, appid and appid
           end
         end
@@ -106,7 +106,6 @@ module ThreeScale
           raise ApplicationNotFound, id unless exists?(service_id, id)
           delete_data service_id, id
           clear_cache service_id, id
-          OAuth::Token::Storage.remove_tokens(service_id, id)
         end
 
         def delete_data(service_id, id)
@@ -157,14 +156,12 @@ module ThreeScale
           )
         end
 
-        def with_app_id_from_params(service_id, app_id, user_key, access_token = nil)
+        def with_app_id_from_params(service_id, app_id, user_key)
           if app_id
             raise AuthenticationError unless user_key.nil?
           elsif user_key
             app_id = load_id_by_key(service_id, user_key)
             raise UserKeyInvalid, user_key if app_id.nil?
-          elsif access_token
-            app_id, * = OAuth::Token::Storage.get_credentials access_token, service_id
           else
             raise ApplicationNotFound
           end

data/lib/3scale/backend/configuration.rb

@@ -1,6 +1,7 @@
 require '3scale/backend/configuration/loader'
 require '3scale/backend/environment'
 require '3scale/backend/configurable'
+require '3scale/backend/errors'
 
 module ThreeScale
   module Backend
@@ -57,7 +58,6 @@ module ThreeScale
       config.add_section(:redshift, :host, :port, :dbname, :user, :password)
       config.add_section(:statsd, :host, :port)
       config.add_section(:internal_api, :user, :password)
-      config.add_section(:oauth, :max_token_size)
       config.add_section(:master, :metrics)
       config.add_section(:worker_prometheus_metrics, :enabled, :port)
       config.add_section(:listener_prometheus_metrics, :enabled, :port)
@@ -78,9 +78,6 @@ module ThreeScale
       master_metrics = [:transactions, :transactions_authorize]
       config.master.metrics = Struct.new(*master_metrics).new
 
-      # Default config
-      config.master_service_id  = 1
-
       # This setting controls whether the listener can create event buckets in
       # Redis. We do not want all the listeners creating buckets yet, as we do
       # not know exactly the rate at which we can send events to Kinesis

data/lib/3scale/backend/errors.rb

@@ -73,36 +73,6 @@ module ThreeScale
       end
     end
 
-    class AccessTokenInvalid < NotFound
-      def initialize(id = nil)
-        super %(token "#{id}" is invalid: expired or never defined)
-      end
-    end
-
-    class AccessTokenAlreadyExists < Error
-      def initialize(id = nil)
-        super %(token "#{id}" already exists)
-      end
-    end
-
-    class AccessTokenStorageError < Error
-      def initialize(id = nil)
-        super %(storage error when saving token "#{id}")
-      end
-    end
-
-    class AccessTokenFormatInvalid < Invalid
-      def initialize
-        super 'token is either too big or has an invalid format'.freeze
-      end
-    end
-
-    class AccessTokenInvalidTTL < Invalid
-      def initialize
-        super 'the specified TTL should be a positive integer'.freeze
-      end
-    end
-
     class ServiceNotActive < Error
       def initialize
         super 'service is not active'.freeze
@@ -182,12 +152,6 @@ module ThreeScale
       end
     end
 
-    class RequiredParamsMissing < Invalid
-      def initialize
-        super 'missing required parameters'.freeze
-      end
-    end
-
     class UsageValueInvalid < Error
       def initialize(metric_name, value)
         if !value.is_a?(String) || value.blank?

data/lib/3scale/backend/listener.rb

@@ -30,9 +30,6 @@ module ThreeScale
       ##~ @parameter_app_id_inline = @parameter_app_id.clone
       ##~ @parameter_app_id_inline["description_inline"] = true
       ##
-      ##~ @parameter_access_token = {"name" => "access_token", "dataType" => "string", "required" => false, "paramType" => "query", "threescale_name" => "access_tokens"}
-      ##~ @parameter_access_token["description"] = "OAuth token used for authorizing if you don't use client_id with client_secret."
-      ##
       ##~ @parameter_client_id = {"name" => "app_id", "dataType" => "string", "required" => false, "paramType" => "query", "threescale_name" => "app_ids"}
       ##~ @parameter_client_id["description"] = "Client Id (identifier of the application if the auth. pattern is OAuth, note that client_id == app_id)"
       ##~ @parameter_client_id_inline = @parameter_client_id.clone
@@ -114,8 +111,7 @@ module ThreeScale
 
 
       AUTH_AUTHREP_COMMON_PARAMS = ['service_id'.freeze, 'app_id'.freeze, 'app_key'.freeze,
-                                    'user_key'.freeze, 'provider_key'.freeze,
-                                    'access_token'.freeze].freeze
+                                    'user_key'.freeze, 'provider_key'.freeze].freeze
       private_constant :AUTH_AUTHREP_COMMON_PARAMS
 
       REPORT_EXPECTED_PARAMS = ['provider_key'.freeze,
@@ -128,8 +124,6 @@ module ThreeScale
         disable :dump_errors
       end
 
-      set :views, File.dirname(__FILE__) + '/views'
-
       use Backend::Rack::ExceptionCatcher
 
       before do
@@ -252,7 +246,7 @@ module ThreeScale
       ##~ op.summary = "Authorize (OAuth authentication mode pattern)"
       ##
       ##~ op.description = "<p>Read-only operation to authorize an application in the OAuth authentication pattern."
-      ##~ @oauth_security = "<p>When using this endpoint please pay attention at your handling of app_id and app_key parameters. If you don't specify an app_key, the endpoint assumes the app_id specified has already been authenticated by other means. If you specify the app_key parameter, even if it is empty, it will be checked against the application's keys. If you don't trust the app_id value you have, either use app keys and specify one or use access_token and avoid the app_id parameter."
+      ##~ @oauth_security = "<p>When using this endpoint please pay attention at your handling of app_id and app_key parameters. If you don't specify an app_key, the endpoint assumes the app_id specified has already been authenticated by other means. If you specify the app_key parameter, even if it is empty, it will be checked against the application's keys. If you don't trust the app_id value you have, use app keys and specify one."
       ##~ @oauth_desc_response = "<p>This call returns extra data (secret and redirect_url) needed to power OAuth APIs. It's only available for users with OAuth enabled APIs."
       ##~ op.description = op.description + @oauth_security + @oauth_desc_response
       ##~ op.description = op.description + " " + @authorize_desc + " " + @authorize_desc_response
@@ -263,7 +257,6 @@ module ThreeScale
       ##
       ##~ op.parameters.add @parameter_service_token
       ##~ op.parameters.add @parameter_service_id
-      ##~ op.parameters.add @parameter_access_token
       ##~ op.parameters.add @parameter_client_id
       ##~ op.parameters.add @parameter_app_key_oauth
       ##~ op.parameters.add @parameter_referrer
@@ -337,7 +330,6 @@ module ThreeScale
       ##
       ##~ op.parameters.add @parameter_service_token
       ##~ op.parameters.add @parameter_service_id
-      ##~ op.parameters.add @parameter_access_token
       ##~ op.parameters.add @parameter_client_id
       ##~ op.parameters.add @parameter_app_key_oauth
       ##~ op.parameters.add @parameter_referrer
@@ -430,62 +422,6 @@ module ThreeScale
         202
       end
 
-      ## OAUTH ACCESS TOKENS
-
-      # These endpoints are deprecated and are going to be removed. For now,
-      # let's disable them.
-      if Backend.test?
-        post '/services/:service_id/oauth_access_tokens.xml' do
-          check_post_content_type!
-          require_params! :service_id, :token
-
-          service_id = params[:service_id]
-          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
-
-          app_id = params[:app_id]
-          raise ApplicationNotFound, app_id unless Application.exists?(service_id, app_id)
-
-          OAuth::Token::Storage.create(params[:token], service_id, app_id, params[:ttl])
-        end
-
-        delete '/services/:service_id/oauth_access_tokens/:token.xml' do
-          require_params! :service_id, :token
-
-          service_id = params[:service_id]
-          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
-
-          token = params[:token]
-
-          # TODO: perhaps improve this to list the deleted tokens?
-          raise AccessTokenInvalid, token unless OAuth::Token::Storage.delete(token, service_id)
-        end
-
-        get '/services/:service_id/applications/:app_id/oauth_access_tokens.xml' do
-          require_params! :service_id, :app_id
-
-          service_id = params[:service_id]
-          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
-
-          app_id = params[:app_id]
-
-          raise ApplicationNotFound, app_id unless Application.exists?(service_id, app_id)
-
-          @tokens = OAuth::Token::Storage.all_by_service_and_app service_id, app_id
-          builder :oauth_access_tokens
-        end
-
-        get '/services/:service_id/oauth_access_tokens/:token.xml' do
-          require_params! :service_id, :token
-
-          service_id = params[:service_id]
-          ensure_authenticated!(params[:provider_key], params[:service_token], service_id)
-
-          @token_to_app_id = OAuth::Token::Storage.get_credentials(params[:token], service_id)
-
-          builder :oauth_app_id_by_token
-        end
-      end
-
       get '/check.txt' do
         content_type 'text/plain'
         body 'ok'
@@ -518,10 +454,6 @@ module ThreeScale
         params[:usage].nil? || params[:usage].is_a?(Hash)
       end
 
-      def require_params!(*keys)
-        raise RequiredParamsMissing unless params && keys.all? { |key| !blank?(params[key]) }
-      end
-
       def check_params_value_encoding!(input_params, params_to_validate)
         params_to_validate.each do |p|
           param_value = input_params[p]
@@ -651,15 +583,6 @@ module ThreeScale
         raise ServiceTokenInvalid.new(token, id)
       end
 
-      def ensure_authenticated!(provider_key, service_token, service_id)
-        if blank?(provider_key)
-          key = provider_key_from(service_token, service_id)
-          raise_provider_key_error(params) if blank?(key)
-        elsif !Service.authenticate_service_id(service_id, provider_key)
-          raise ProviderKeyInvalid, provider_key
-        end
-      end
-
       def response_auth_call(auth_status)
         status(auth_status.authorized? ? 200 : 409)
         optionally_set_headers(auth_status)

data/lib/3scale/backend/listener_metrics.rb

@@ -4,14 +4,34 @@ require 'rack'
 module ThreeScale
   module Backend
     class ListenerMetrics
-      REQUEST_TYPES = {
+      AUTH_AND_REPORT_REQUEST_TYPES = {
         '/transactions/authorize.xml' => 'authorize',
         '/transactions/oauth_authorize.xml' => 'authorize_oauth',
         '/transactions/authrep.xml' => 'authrep',
         '/transactions/oauth_authrep.xml' => 'authrep_oauth',
         '/transactions.xml' => 'report'
       }
-      private_constant :REQUEST_TYPES
+      private_constant :AUTH_AND_REPORT_REQUEST_TYPES
+
+      INTERNAL_API_PATHS = [
+        [/\/services\/.*\/alert_limits/, 'alerts'],
+        [/\/services\/.*\/applications\/.*\/keys/, 'application_keys'],
+        [/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'],
+        [/\/services\/.*\/applications/, 'applications'],
+        [/\/services\/.*\/errors/, 'errors'],
+        [/\/events/, 'events'],
+        [/\/services\/.*\/metrics/, 'metrics'],
+        [/\/service_tokens/, 'service_tokens'],
+        [/\/services/, 'services'],
+        [/\/services\/.*\/stats/, 'stats'],
+        [/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'],
+        [/\/services\/.*\/applications\/.*\/utilization/, 'utilization'],
+      ].freeze
+      private_constant :INTERNAL_API_PATHS
+
+      # Most requests will be under 100ms, so use a higher granularity from there
+      TIME_BUCKETS = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+      private_constant :TIME_BUCKETS
 
       class << self
         ERRORS_4XX_TO_TRACK = Set[403, 404, 409].freeze
@@ -27,9 +47,12 @@ module ThreeScale
         end
 
         def report_resp_code(path, resp_code)
-          Yabeda.apisonator_listener.response_codes.increment(
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_codes.increment(
             {
-              request_type: REQUEST_TYPES[path],
+              request_type: req_type,
               resp_code: code_group(resp_code)
             },
             by: 1
@@ -37,8 +60,11 @@ module ThreeScale
         end
 
         def report_response_time(path, request_time)
-          Yabeda.apisonator_listener.response_times.measure(
-            { request_type: REQUEST_TYPES[path] },
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_times.measure(
+            { request_type: req_type },
             request_time
           )
         end
@@ -69,8 +95,21 @@ module ThreeScale
                 comment 'Response times'
                 unit :seconds
                 tags %i[request_type]
-                # Most requests will be under 100ms, so use a higher granularity from there
-                buckets [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+                buckets TIME_BUCKETS
+              end
+            end
+
+            group :apisonator_listener_internal_api do
+              counter :response_codes do
+                comment 'Response codes'
+                tags %i[request_type resp_code]
+              end
+
+              histogram :response_times do
+                comment 'Response times'
+                unit :seconds
+                tags %i[request_type]
+                buckets TIME_BUCKETS
               end
             end
           end
@@ -93,6 +132,24 @@ module ThreeScale
             'unknown'.freeze
           end
         end
+
+        def req_type(path)
+          AUTH_AND_REPORT_REQUEST_TYPES[path] || internal_api_req_type(path)
+        end
+
+        def internal_api_req_type(path)
+          (_regex, type) = INTERNAL_API_PATHS.find { |(regex, _)| regex.match path }
+          type
+        end
+
+        # Returns the group as defined in .define_metrics
+        def prometheus_group(request_type)
+          if AUTH_AND_REPORT_REQUEST_TYPES.values.include? request_type
+            :apisonator_listener
+          else
+            :apisonator_listener_internal_api
+          end
+        end
       end
     end
   end

data/lib/3scale/backend/rack/prometheus.rb

@@ -8,7 +8,15 @@ module ThreeScale
 
         def call(env)
           began_at = Time.now.getutc
-          status, header, body = @app.call(env)
+
+          begin
+            status, header, body = @app.call(env)
+          rescue Exception => e
+            ListenerMetrics.report_resp_code(env['REQUEST_PATH'], 500)
+            ListenerMetrics.report_response_time(env['REQUEST_PATH'], Time.now - began_at)
+            raise e
+          end
+
           ListenerMetrics.report_resp_code(env['REQUEST_PATH'], status)
           ListenerMetrics.report_response_time(env['REQUEST_PATH'], Time.now - began_at)
           [status, header, body]

data/lib/3scale/backend/storage_async/client.rb

@@ -47,7 +47,7 @@ module ThreeScale
 
           endpoint = Async::IO::Endpoint.tcp(host, port)
           @redis_async = Async::Redis::Client.new(
-            endpoint, connection_limit: opts[:max_connections]
+            endpoint, limit: opts[:max_connections]
           )
           @building_pipeline = false
         end

data/lib/3scale/backend/transactor.rb

@@ -65,19 +65,7 @@ module ThreeScale
         params[:app_id] = nil if app_id && app_id.empty?
 
         if oauth
-          if app_id.nil?
-            access_token = params[:access_token]
-            access_token = nil if access_token && access_token.empty?
-
-            if access_token.nil?
-              raise ApplicationNotFound.new nil if app_id.nil?
-            else
-              app_id = get_token_ids(access_token, service_id, app_id)
-              # update params, since they are checked elsewhere
-              params[:app_id] = app_id
-            end
-          end
-
+          raise ApplicationNotFound.new nil if app_id.nil?
           validators = Validators::OAUTH_VALIDATORS
         else
           validators = Validators::VALIDATORS

data/lib/3scale/backend/transactor/notify_batcher.rb

@@ -30,9 +30,13 @@ module ThreeScale
         end
 
         def notify(provider_key, usage)
-          # batch several notifications together so that we can process just one
+          # We need the master service ID to report its metrics. If it's not
+          # set, we don't need to notify anything.
+          # Batch several notifications together so that we can process just one
           # job for a group of them.
-          notify_batch(provider_key, usage)
+          unless configuration.master_service_id.to_s.empty?
+            notify_batch(provider_key, usage)
+          end
         end
 
         def notify_batch(provider_key, usage)

data/lib/3scale/backend/transactor/notify_job.rb

@@ -7,8 +7,6 @@ module ThreeScale
         extend Configurable
         @queue = :main
 
-        InvalidMasterServiceId = Class.new(ThreeScale::Backend::Error)
-
         class << self
           def perform_logged(provider_key, usage, timestamp, _enqueue_time)
             application_id = Application.load_id_by_key(master_service_id, provider_key)
@@ -16,12 +14,42 @@ module ThreeScale
             if application_id && Application.exists?(master_service_id, application_id)
               master_metrics = Metric.load_all(master_service_id)
 
-              ProcessJob.perform([{
-                service_id: master_service_id,
-                application_id: application_id,
-                timestamp: timestamp,
-                usage: master_metrics.process_usage(usage)
-              }])
+              begin
+                ProcessJob.perform([{
+                  service_id: master_service_id,
+                  application_id: application_id,
+                  timestamp: timestamp,
+                  usage: master_metrics.process_usage(usage)
+                }])
+              rescue MetricInvalid => e
+                # This happens when the master account in Porta does not have
+                # the notify metrics defined (by default "transactions" and
+                # "transactions/authorize"). These metrics need to be created in
+                # Porta, Apisonator does not have a way to guarantee that
+                # they're defined.
+                # Notice that this rescue prevents the job from being retried.
+                # Apisonator can't know when the metrics will be created (if
+                # ever) so it's better to log the error rather than retrying
+                # these jobs for an undefined period of time.
+                Worker.logger.notify(e)
+                return [false, "#{e}"]
+              rescue TransactionTimestampNotWithinRange => e
+                # This is very unlikely to happen. The timestamps in a notify
+                # job are not set by users, they are set by the listeners. If
+                # this error happens it might mean that:
+                # a) The worker started processing this job way after the
+                # listener produced it. This can happen for example if we make
+                # some requests to a listener with no workers. The listeners
+                # will enqueue some notify jobs. If we start a worker hours
+                # later, we might see this error.
+                # b) There's some kind of clock skew issue.
+                # c) There's a bug.
+                #
+                # We can't raise here, because then, the job will be retried,
+                # but it's going to fail always if it has an old timestamp.
+                Worker.logger.notify(e)
+                return [false, "#{provider_key} #{application_id} #{e}"]
+              end
             end
             [true, "#{provider_key} #{application_id || '--'}"]
           end
@@ -29,15 +57,7 @@ module ThreeScale
           private
 
           def master_service_id
-            value = configuration.master_service_id
-
-            unless value
-              raise InvalidMasterServiceId,
-                    "Can't find master service id. Make sure the \"master_service_id\" "\
-                    'configuration value is set correctly'
-            end
-
-            value.to_s
+            configuration.master_service_id.to_s
           end
         end
       end

data/lib/3scale/backend/transactor/report_job.rb

@@ -59,7 +59,7 @@ module ThreeScale
             return [] if transactions.empty?
             transactions = transactions.values if transactions.respond_to?(:values)
             transactions.group_by do |transaction|
-              Application.extract_id!(service_id, transaction['app_id'], transaction['user_key'], transaction['access_token'])
+              Application.extract_id!(service_id, transaction['app_id'], transaction['user_key'])
             end.each(&block)
           end
 

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '2.101.1'
+    VERSION = '3.1.0'
   end
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>2.101.1</version>
+        <version>3.1.0</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -73,7 +73,7 @@
       </dependency>
           <dependency>
         <packageName>async-io</packageName>
-        <version>1.27.5</version>
+        <version>1.27.7</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -93,7 +93,7 @@
       </dependency>
           <dependency>
         <packageName>async-redis</packageName>
-        <version>0.4.1</version>
+        <version>0.5.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -371,7 +371,7 @@
       </dependency>
           <dependency>
         <packageName>json</packageName>
-        <version>2.1.0</version>
+        <version>2.3.1</version>
         <licenses>
                       <license>
               <name>ruby</name>
@@ -669,7 +669,7 @@
       </dependency>
           <dependency>
         <packageName>protocol-redis</packageName>
-        <version>0.2.0</version>
+        <version>0.5.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -719,7 +719,7 @@
       </dependency>
           <dependency>
         <packageName>rack</packageName>
-        <version>2.0.9</version>
+        <version>2.1.4</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 2.101.1
+  version: 3.1.0
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-05 00:00:00.000000000 Z
+date: 2020-10-14 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.
@@ -111,11 +111,6 @@ files:
 - lib/3scale/backend/memoizer.rb
 - lib/3scale/backend/metric.rb
 - lib/3scale/backend/metric/collection.rb
-- lib/3scale/backend/oauth.rb
-- lib/3scale/backend/oauth/token.rb
-- lib/3scale/backend/oauth/token_key.rb
-- lib/3scale/backend/oauth/token_storage.rb
-- lib/3scale/backend/oauth/token_value.rb
 - lib/3scale/backend/period.rb
 - lib/3scale/backend/period/boundary.rb
 - lib/3scale/backend/period/cache.rb
@@ -184,8 +179,6 @@ files:
 - lib/3scale/backend/validators/service_state.rb
 - lib/3scale/backend/validators/state.rb
 - lib/3scale/backend/version.rb
-- lib/3scale/backend/views/oauth_access_tokens.builder
-- lib/3scale/backend/views/oauth_app_id_by_token.builder
 - lib/3scale/backend/worker.rb
 - lib/3scale/backend/worker_async.rb
 - lib/3scale/backend/worker_metrics.rb

data/lib/3scale/backend/oauth.rb

@@ -1,4 +0,0 @@
-require '3scale/backend/oauth/token'
-require '3scale/backend/oauth/token_key'
-require '3scale/backend/oauth/token_value'
-require '3scale/backend/oauth/token_storage'

data/lib/3scale/backend/oauth/token.rb

@@ -1,26 +0,0 @@
-module ThreeScale
-  module Backend
-    module OAuth
-      class Token
-        attr_reader :service_id, :token, :key
-        attr_accessor :ttl, :app_id
-
-        def initialize(token, service_id, app_id, ttl)
-          @token = token
-          @service_id = service_id
-          @app_id = app_id
-          @ttl = ttl
-          @key = Key.for token, service_id
-        end
-
-        def value
-          Value.for app_id
-        end
-
-        def self.from_value(token, service_id, value, ttl)
-          new token, service_id, *Value.from(value), ttl
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/oauth/token_key.rb

@@ -1,30 +0,0 @@
-# This module defines the format of the keys for OAuth tokens and token sets.
-#
-# Note that while we can build the key easily, we cannot reliably obtain a token
-# and a service_id out of the key, because there are no constraints on them:
-#
-# "oauth_access_tokens/service:some/servicegoeshere/andthisis_a_/valid_token"
-#
-module ThreeScale
-  module Backend
-    module OAuth
-      class Token
-        module Key
-          class << self
-            def for(token, service_id)
-              "oauth_access_tokens/service:#{service_id}/#{token}"
-            end
-          end
-
-          module Set
-            class << self
-              def for(service_id, app_id)
-                "oauth_access_tokens/service:#{service_id}/app:#{app_id}/"
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/oauth/token_storage.rb

@@ -1,313 +0,0 @@
-module ThreeScale
-  module Backend
-    module OAuth
-      class Token
-        module Storage
-          include Configurable
-
-          # Default token size is 4K - 512 (to allow for some metadata)
-          MAXIMUM_TOKEN_SIZE = configuration.oauth.max_token_size || 3584
-          private_constant :MAXIMUM_TOKEN_SIZE
-          TOKEN_MAX_REDIS_SLICE_SIZE = 500
-          private_constant :TOKEN_MAX_REDIS_SLICE_SIZE
-          TOKEN_TTL_DEFAULT = 86400
-          private_constant :TOKEN_TTL_DEFAULT
-          TOKEN_TTL_PERMANENT = 0
-          private_constant :TOKEN_TTL_PERMANENT
-
-          Error = Class.new StandardError
-          InconsistencyError = Class.new Error
-
-          class << self
-            include Backend::Logging
-            include Backend::StorageHelpers
-
-            def create(token, service_id, app_id, ttl = nil)
-              raise AccessTokenFormatInvalid if token.nil? || token.empty? ||
-                !token.is_a?(String) || token.bytesize > MAXIMUM_TOKEN_SIZE
-
-              # raises if TTL is invalid
-              ttl = sanitized_ttl ttl
-
-              key = Key.for token, service_id
-              raise AccessTokenAlreadyExists.new(token) unless storage.get(key).nil?
-
-              value = Value.for app_id
-              token_set = Key::Set.for(service_id, app_id)
-
-              store_token token, token_set, key, value, ttl
-              ensure_stored! token, token_set, key, value
-            end
-
-            # Deletes a token
-            #
-            # Returns the associated app_id or nil
-            #
-            def delete(token, service_id)
-              key = Key.for token, service_id
-              val = storage.get key
-              if val
-                app_id = Value.from val
-                token_set = Key::Set.for(service_id, app_id)
-
-                existed, * = remove_a_token token_set, token, key
-
-                unless existed
-                  logger.notify(InconsistencyError.new("Found OAuth token " \
-                    "#{token} for service #{service_id} and app #{app_id} as " \
-                    "key but not in set!"))
-                end
-              end
-
-              val
-            end
-
-            # Get a token's associated app_id
-            def get_credentials(token, service_id)
-              app_id = Value.from(storage.get(Key.for(token, service_id)))
-              raise AccessTokenInvalid.new token if app_id.nil?
-              app_id
-            end
-
-            # This is used to list tokens by service and app.
-            #
-            # Note: this deletes tokens that have not been found from the set of
-            # tokens for the given app - those have to be expired tokens.
-            def all_by_service_and_app(service_id, app_id)
-              token_set = Key::Set.for(service_id, app_id)
-              deltokens = []
-              tokens_n_values_flat(token_set, service_id)
-                .select do |(token, _key, value, _ttl)|
-                  app_id = Value.from value
-                  if app_id.nil?
-                    deltokens << token
-                    false
-                  else
-                    true
-                  end
-                end
-                .map do |(token, _key, value, ttl)|
-                  Token.from_value token, service_id, value, ttl
-                end
-                .tap do
-                  # delete expired tokens (nil values) from token set
-                  deltokens.each_slice(TOKEN_MAX_REDIS_SLICE_SIZE) do |delgrp|
-                    storage.srem token_set, delgrp
-                  end
-                end
-            end
-
-            # Remove tokens by app_id.
-            #
-            # Triggered by Application deletion.
-            #
-            def remove_tokens(service_id, app_id)
-              remove_tokens_by service_id, app_id
-            end
-
-
-            private
-
-            # Remove all tokens
-            #
-            # I thought of leaving this one public, but remove_*_tokens removed
-            # my use cases for the time being.
-            def remove_tokens_by(service_id, app_id)
-              token_set = Key::Set.for(service_id, app_id)
-
-              remove_whole_token_set(token_set, service_id)
-            end
-
-            def remove_token_set_by(token_set, service_id, &blk)
-              # Get tokens. Filter them. Group them into manageable groups.
-              # Extract tokens and keys into separate arrays, one for each.
-              # Remove tokens from token set (they are keys in a set) and token
-              # keys themselves.
-              tokens_n_values_flat(token_set, service_id, false)
-              .select(&blk)
-              .each_slice(TOKEN_MAX_REDIS_SLICE_SIZE)
-              .inject([[], []]) do |acc, groups|
-                groups.each do |token, key, _value|
-                  acc[0] << token
-                  acc[1] << key
-                end
-                acc
-              end
-              .each_slice(2)
-              .inject([]) do |acc, (tokens, keys)|
-                storage.pipelined do
-                  if tokens && !tokens.empty?
-                    storage.srem token_set, tokens
-                    acc.concat tokens
-                  end
-                  storage.del keys if keys && !keys.empty?
-                end
-                acc
-              end
-            end
-
-            def remove_a_token(token_set, token, key)
-              storage.pipelined do
-                storage.srem token_set, token
-                storage.del key
-              end
-            end
-
-            def remove_whole_token_set(token_set, service_id)
-              _token_groups, key_groups = tokens_n_keys(token_set, service_id)
-              storage.pipelined do
-                storage.del token_set
-                # remove all tokens for this app
-                key_groups.each do |keys|
-                  storage.del keys
-                end
-              end
-            end
-
-            # TODO: provide a SSCAN interface with lazy enums because SMEMBERS
-            # is prone to DoSing and timeouts
-            def tokens_from(token_set)
-              storage.smembers(token_set)
-            end
-
-            def tokens_n_keys(token_set, service_id)
-              token_groups = tokens_from(token_set).each_slice(TOKEN_MAX_REDIS_SLICE_SIZE)
-              key_groups = token_groups.map do |tokens|
-                tokens.map do |token|
-                  Key.for token, service_id
-                end
-              end
-
-              [token_groups, key_groups]
-            end
-
-            # Provides grouped data which matches respectively in each array
-            # position, ie. 1st group of data contains a group of tokens, keys
-            # and values with ttls, and position N of the tokens group has key
-            # in position N of the keys group, and so on.
-            #
-            # [[[token group], [key group], [value_with_ttls_group]], ...]
-            #
-            def tokens_n_values_groups(token_set, service_id, with_ttls)
-              token_groups, key_groups = tokens_n_keys(token_set, service_id)
-              value_ttl_groups = key_groups.map do |keys|
-                # pipelining will create an array with the results of commands
-                res = storage.pipelined do
-                  storage.mget(keys)
-                  if with_ttls
-                    keys.map do |key|
-                      storage.ttl key
-                    end
-                  end
-                end
-                # [mget array, 0..n ttls] => [mget array, ttls array]
-                [res.shift, res]
-              end
-              token_groups.zip(key_groups, value_ttl_groups)
-            end
-
-            # Zips the data provided by tokens_n_values_groups so that you stop
-            # looking at indexes in the respective arrays and instead have:
-            #
-            # [group 0, ..., group N] where each group is made of:
-            #   [[token 0, key 0, value 0, ttl 0], ..., [token N, key N, value
-            #   N, ttl N]]
-            #
-            def tokens_n_values_zipped_groups(token_set, service_id, with_ttls = true)
-              tokens_n_values_groups(token_set,
-                                     service_id,
-                                     with_ttls).map do |tokens, keys, (values, ttls)|
-                tokens.zip keys, values, ttls
-              end
-            end
-
-            # Flattens the data provided by tokens_n_values_zipped_groups so
-            # that you have a transparent iterator with all needed data and can
-            # stop worrying about streaming groups of elements.
-            #
-            def tokens_n_values_flat(token_set, service_id, with_ttls = true)
-              tokens_n_values_zipped_groups(token_set,
-                                            service_id,
-                                            with_ttls).flat_map do |groups|
-                groups.map do |token, key, value, ttl|
-                  [token, key, value, ttl]
-                end
-              end
-            end
-
-            # Store the specified token in Redis
-            #
-            # TTL specified in seconds.
-            # A TTL of 0 stores a permanent token
-            def store_token(token, token_set, key, value, ttl)
-              # build the storage command so that we can pipeline everything cleanly
-              command = :set
-              args = [key]
-
-              if !permanent_ttl? ttl
-                command = :setex
-                args << ttl
-              end
-
-              args << value
-
-              # pipelined will return nil if it is embedded into another
-              # pipeline(which would be an error at this point) or if shutting
-              # down and a connection error happens. Both things being abnormal
-              # means we should just raise a storage error.
-              raise AccessTokenStorageError, token unless storage.pipelined do
-                storage.send(command, *args)
-                storage.sadd(token_set, token)
-              end
-            end
-
-
-            # Make sure everything ended up there
-            #
-            # TODO: review and possibly reimplement trying to leave it
-            # consistent as much as possible.
-            #
-            # Note that we have a sharding proxy and pipelines can't be guaranteed
-            # to behave like transactions, since we might have one non-working
-            # shard. Instead of relying on proxy-specific responses, we just check
-            # that the data we should have in the store is really there.
-            def ensure_stored!(token, token_set, key, value)
-              results = storage.pipelined do
-                storage.get(key)
-                storage.sismember(token_set, token)
-              end
-
-              results.last && results.first == value ||
-                raise(AccessTokenStorageError, token)
-            end
-
-	    # Validation for the TTL value
-	    #
-	    # 0 is accepted (understood as permanent token)
-	    # Negative values are not accepted
-	    # Integer(ttl) validation is required (if input is nil, default applies)
-	    def sanitized_ttl(ttl)
-              ttl = begin
-                      Integer(ttl)
-                    rescue TypeError
-                      # ttl is nil
-                      TOKEN_TTL_DEFAULT
-                    rescue
-                      # NaN
-                      -1
-                    end
-              raise AccessTokenInvalidTTL if ttl < 0
-
-	      ttl
-            end
-
-	    # Check whether a TTL has the magic value for a permanent token
-	    def permanent_ttl?(ttl)
-              ttl == TOKEN_TTL_PERMANENT
-	    end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/oauth/token_value.rb

@@ -1,25 +0,0 @@
-# This module encodes values in Redis for our tokens
-module ThreeScale
-  module Backend
-    module OAuth
-      class Token
-        module Value
-          # Note: this module made more sense when it also supported end-users.
-          # Given how simple the module is, we could get rid of it in a future
-          # refactor.
-
-          class << self
-            # this method is used when creating tokens
-            def for(app_id)
-              app_id
-            end
-
-            def from(value)
-              value
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/3scale/backend/views/oauth_access_tokens.builder

@@ -1,14 +0,0 @@
-xml.instruct!
-xml.oauth_access_tokens do
-  @tokens.each do |t|
-    # Some bright head leaked the -1 that Redis uses to indicate there is no TTL
-    # associated with a key. The behaviour is inconsistent because somehow we
-    # BOTH expect that a token with no TTL does not have a "ttl" attribute in
-    # the generated XML and at the same time we expect (in some tests) that such
-    # tokens have this field with a -1 value.
-    #
-    # Just enforce the ttl to be -1 when there is none.
-    attrs = { :ttl => t.ttl || -1 }
-    xml.oauth_access_token t.token, attrs
-  end
-end

data/lib/3scale/backend/views/oauth_app_id_by_token.builder

@@ -1,4 +0,0 @@
-xml.instruct!
-xml.application do
-  xml.app_id @token_to_app_id
-end