apisonator 3.0.0.pre1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0fa79b460b2906815bfb027b35522ea438f4ecb8952643c2c66b886a28dcff09
-  data.tar.gz: 491656a6738201be52b2d3062b916db4f52cb38dd3edc8a2ba2b0c5816412a73
+  metadata.gz: 1036d32ab3793674c8fe3d815e471581d0056c9a7739b46079ce67f89827bc40
+  data.tar.gz: 63bd9f6b690a4e5ec8fa276839001979cbf25f38e9105727679dfcb6d962a0ea
 SHA512:
-  metadata.gz: 75f2a8668f08c65ea4b125c1a1c86939f1e75e7d8b0d0aa5c2c084881fbe380a7eed3a8863d391f029d78460fd5f3c8036ec8c208177dd1fda813808577ed351
-  data.tar.gz: eb6f93ad8861cb4916afa819a8d2bc35db94b6c240c34048dd5e6ec5039ae7ade2977b3d0c251b86a63e710a73f1918785ecfd12f195f6e1dfb4d7bf12004fb4
+  metadata.gz: 669fa971a4f210d6709f40bac15bad1768e803f5bd4b979a66046151e8f8b572735088dd4e164866edafba681b6de7311abcce4bab9bb951e3b7fcfc128e1cb4
+  data.tar.gz: 58775cf983c7ba17ff823a9f11d41ef7790b63c3f206cf718f4ba605be8781bc5c8fe99830b8e2bb9da4951dfd7108655b83e055f99bc8c91d6950c891f0234b

data/CHANGELOG.md

@@ -2,6 +2,75 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.2.0 - 2021-01-19
+
+### Added
+
+- New endpoint in the internal API to get the provider key for a given (token,
+service_id) pair ([#243](https://github.com/3scale/apisonator/pull/243)).
+
+### Changed
+
+- The config file used when running in a Docker image now parses "1" and "true"
+(case-insensitive) as true
+([#245](https://github.com/3scale/apisonator/pull/245)).
+
+### Fixed
+
+- Fixed some metrics of the internal API that were not being counted
+correctly([#244](https://github.com/3scale/apisonator/pull/244)).
+
+
+## 3.1.0 - 2020-10-14
+
+### Added
+
+- Prometheus metrics for the internal API
+([#236](https://github.com/3scale/apisonator/pull/236)).
+- Docs with a detailed explanation about how counter updates are performed
+([#239](https://github.com/3scale/apisonator/pull/239)).
+
+### Changed
+
+- NotifyJobs are run only when the service ID is explicitly defined
+([#238](https://github.com/3scale/apisonator/pull/238)).
+
+### Fixed
+
+- Fixed corner case that raised "TransactionTimestampNotWithinRange" in notify
+jobs ([#235](https://github.com/3scale/apisonator/pull/235)).
+
+
+## 3.0.1.1 - 2020-07-28
+
+### Changed
+
+- Updated json gem to v2.3.1
+([#232](https://github.com/3scale/apisonator/pull/232)).
+
+## 3.0.1 - 2020-07-14
+
+### Fixed
+
+- The Prometheus counter of "5xx" errors has been fixed
+([#230](https://github.com/3scale/apisonator/pull/230)).
+
+### Changed
+
+- Gem updates: rack to v2.1.4
+([#228](https://github.com/3scale/apisonator/pull/228)) and async-redis to
+v0.5.0 ([#229](https://github.com/3scale/apisonator/pull/229)).
+
+
+## 3.0.0 - 2020-06-19
+
+### Removed
+
+- Apisonator no longer supports the native oauth authorization mode. This is a
+feature that was deprecated a long time ago
+([#226](https://github.com/3scale/apisonator/pull/226)).
+
+
 ## 2.101.1 - 2020-06-05
 
 ### Fixed

data/Gemfile.base

@@ -53,13 +53,13 @@ gem 'rake', '~> 13.0'
 gem 'builder', '= 3.2.3'
 # Use a patched resque to allow reusing their Airbrake Failure class
 gem 'resque', git: 'https://github.com/3scale/resque', branch: '3scale'
-gem 'rack', '~> 2.0.8'
+gem 'rack', '~> 2.1.4'
 gem 'sinatra', '~> 2.0.3'
 gem 'sinatra-contrib', '~> 2.0.3'
 # Optional external error logging services
 gem 'bugsnag', '~> 6', require: nil
 gem 'yabeda-prometheus', '~> 0.5.0'
-gem 'async-redis', '~> 0.4.1'
+gem 'async-redis', '~> 0.5'
 gem 'falcon', '~> 0.35'
 
 # Use a patched redis-rb that fixes an issue when trying to connect with

data/Gemfile.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.0.pre1)
+    apisonator (3.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -66,14 +66,15 @@ GEM
     async-http-cache (0.1.5)
       async-http
       protocol-http (~> 0.14)
-    async-io (1.27.5)
+    async-io (1.27.7)
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.4.1)
+    async-redis (0.5.0)
       async (~> 1.8)
       async-io (~> 1.10)
-      protocol-redis (~> 0.2.0)
+      async-pool (~> 0.2)
+      protocol-redis (~> 0.5.0)
     async-rspec (1.13.0)
       rspec (~> 3.0)
       rspec-files (~> 1.0)
@@ -118,7 +119,7 @@ GEM
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jmespath (1.3.1)
-    json (2.1.0)
+    json (2.3.1)
     license_finder (5.9.2)
       bundler
       rubyzip
@@ -161,7 +162,7 @@ GEM
     protocol-http2 (0.11.6)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
-    protocol-redis (0.2.0)
+    protocol-redis (0.5.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -171,7 +172,7 @@ GEM
     pry-doc (0.11.1)
       pry (~> 0.9)
       yard (~> 0.9)
-    rack (2.0.9)
+    rack (2.1.4)
     rack-protection (2.0.3)
       rack
     rack-test (0.8.2)
@@ -266,7 +267,7 @@ PLATFORMS
 DEPENDENCIES
   airbrake (= 4.3.1)
   apisonator!
-  async-redis (~> 0.4.1)
+  async-redis (~> 0.5)
   async-rspec
   aws-sdk (= 2.4.2)
   benchmark-ips (~> 2.7.2)
@@ -286,7 +287,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   pry-doc (~> 0.11.1)
   puma!
-  rack (~> 2.0.8)
+  rack (~> 2.1.4)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
   redis!

data/Gemfile.on_prem.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.0.pre1)
+    apisonator (3.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -63,14 +63,15 @@ GEM
     async-http-cache (0.1.5)
       async-http
       protocol-http (~> 0.14)
-    async-io (1.27.5)
+    async-io (1.27.7)
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.4.1)
+    async-redis (0.5.0)
       async (~> 1.8)
       async-io (~> 1.10)
-      protocol-redis (~> 0.2.0)
+      async-pool (~> 0.2)
+      protocol-redis (~> 0.5.0)
     async-rspec (1.13.0)
       rspec (~> 3.0)
       rspec-files (~> 1.0)
@@ -107,7 +108,7 @@ GEM
     hiredis (0.6.3)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
+    json (2.3.1)
     license_finder (5.9.2)
       bundler
       rubyzip
@@ -149,7 +150,7 @@ GEM
     protocol-http2 (0.11.6)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
-    protocol-redis (0.2.0)
+    protocol-redis (0.5.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -159,7 +160,7 @@ GEM
     pry-doc (0.11.1)
       pry (~> 0.9)
       yard (~> 0.9)
-    rack (2.0.9)
+    rack (2.1.4)
     rack-protection (2.0.3)
       rack
     rack-test (0.8.2)
@@ -249,7 +250,7 @@ PLATFORMS
 
 DEPENDENCIES
   apisonator!
-  async-redis (~> 0.4.1)
+  async-redis (~> 0.5)
   async-rspec
   benchmark-ips (~> 2.7.2)
   bugsnag (~> 6)
@@ -267,7 +268,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   pry-doc (~> 0.11.1)
   puma!
-  rack (~> 2.0.8)
+  rack (~> 2.1.4)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
   redis!

data/app/api/internal/service_tokens.rb

@@ -7,6 +7,14 @@ module ThreeScale
           ServiceToken.exists?(token, service_id) ? 200 : 404
         end
 
+        get '/:token/:service_id/provider_key' do |token, service_id|
+          if ServiceToken.exists?(token, service_id)
+            { status: :found, provider_key: Service.provider_key_for(service_id) }.to_json
+          else
+            respond_with_404('token/service combination not found'.freeze)
+          end
+        end
+
         post '/' do
           check_tokens_param!
 

data/lib/3scale/backend/configuration.rb

@@ -1,6 +1,7 @@
 require '3scale/backend/configuration/loader'
 require '3scale/backend/environment'
 require '3scale/backend/configurable'
+require '3scale/backend/errors'
 
 module ThreeScale
   module Backend
@@ -77,9 +78,6 @@ module ThreeScale
       master_metrics = [:transactions, :transactions_authorize]
       config.master.metrics = Struct.new(*master_metrics).new
 
-      # Default config
-      config.master_service_id  = 1
-
       # This setting controls whether the listener can create event buckets in
       # Redis. We do not want all the listeners creating buckets yet, as we do
       # not know exactly the rate at which we can send events to Kinesis

data/lib/3scale/backend/listener_metrics.rb

@@ -4,14 +4,36 @@ require 'rack'
 module ThreeScale
   module Backend
     class ListenerMetrics
-      REQUEST_TYPES = {
+      AUTH_AND_REPORT_REQUEST_TYPES = {
         '/transactions/authorize.xml' => 'authorize',
         '/transactions/oauth_authorize.xml' => 'authorize_oauth',
         '/transactions/authrep.xml' => 'authrep',
         '/transactions/oauth_authrep.xml' => 'authrep_oauth',
         '/transactions.xml' => 'report'
       }
-      private_constant :REQUEST_TYPES
+      private_constant :AUTH_AND_REPORT_REQUEST_TYPES
+
+      # Only the first match is taken into account, that's why for example,
+      # "/\/services\/.*\/stats/" needs to appear before "/\/services/"
+      INTERNAL_API_PATHS = [
+        [/\/services\/.*\/alert_limits/, 'alerts'.freeze],
+        [/\/services\/.*\/applications\/.*\/keys/, 'application_keys'.freeze],
+        [/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'.freeze],
+        [/\/services\/.*\/applications\/.*\/utilization/, 'utilization'.freeze],
+        [/\/services\/.*\/applications/, 'applications'.freeze],
+        [/\/services\/.*\/errors/, 'errors'.freeze],
+        [/\/events/, 'events'.freeze],
+        [/\/services\/.*\/metrics/, 'metrics'.freeze],
+        [/\/service_tokens/, 'service_tokens'.freeze],
+        [/\/services\/.*\/stats/, 'stats'.freeze],
+        [/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'.freeze],
+        [/\/services/, 'services'.freeze],
+      ].freeze
+      private_constant :INTERNAL_API_PATHS
+
+      # Most requests will be under 100ms, so use a higher granularity from there
+      TIME_BUCKETS = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+      private_constant :TIME_BUCKETS
 
       class << self
         ERRORS_4XX_TO_TRACK = Set[403, 404, 409].freeze
@@ -27,9 +49,12 @@ module ThreeScale
         end
 
         def report_resp_code(path, resp_code)
-          Yabeda.apisonator_listener.response_codes.increment(
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_codes.increment(
             {
-              request_type: REQUEST_TYPES[path],
+              request_type: req_type,
               resp_code: code_group(resp_code)
             },
             by: 1
@@ -37,8 +62,11 @@ module ThreeScale
         end
 
         def report_response_time(path, request_time)
-          Yabeda.apisonator_listener.response_times.measure(
-            { request_type: REQUEST_TYPES[path] },
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_times.measure(
+            { request_type: req_type },
             request_time
           )
         end
@@ -69,8 +97,21 @@ module ThreeScale
                 comment 'Response times'
                 unit :seconds
                 tags %i[request_type]
-                # Most requests will be under 100ms, so use a higher granularity from there
-                buckets [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+                buckets TIME_BUCKETS
+              end
+            end
+
+            group :apisonator_listener_internal_api do
+              counter :response_codes do
+                comment 'Response codes'
+                tags %i[request_type resp_code]
+              end
+
+              histogram :response_times do
+                comment 'Response times'
+                unit :seconds
+                tags %i[request_type]
+                buckets TIME_BUCKETS
               end
             end
           end
@@ -93,6 +134,24 @@ module ThreeScale
             'unknown'.freeze
           end
         end
+
+        def req_type(path)
+          AUTH_AND_REPORT_REQUEST_TYPES[path] || internal_api_req_type(path)
+        end
+
+        def internal_api_req_type(path)
+          (_regex, type) = INTERNAL_API_PATHS.find { |(regex, _)| regex.match path }
+          type
+        end
+
+        # Returns the group as defined in .define_metrics
+        def prometheus_group(request_type)
+          if AUTH_AND_REPORT_REQUEST_TYPES.values.include? request_type
+            :apisonator_listener
+          else
+            :apisonator_listener_internal_api
+          end
+        end
       end
     end
   end

data/lib/3scale/backend/rack/prometheus.rb

@@ -8,7 +8,15 @@ module ThreeScale
 
         def call(env)
           began_at = Time.now.getutc
-          status, header, body = @app.call(env)
+
+          begin
+            status, header, body = @app.call(env)
+          rescue Exception => e
+            ListenerMetrics.report_resp_code(env['REQUEST_PATH'], 500)
+            ListenerMetrics.report_response_time(env['REQUEST_PATH'], Time.now - began_at)
+            raise e
+          end
+
           ListenerMetrics.report_resp_code(env['REQUEST_PATH'], status)
           ListenerMetrics.report_response_time(env['REQUEST_PATH'], Time.now - began_at)
           [status, header, body]

data/lib/3scale/backend/storage_async/client.rb

@@ -47,7 +47,7 @@ module ThreeScale
 
           endpoint = Async::IO::Endpoint.tcp(host, port)
           @redis_async = Async::Redis::Client.new(
-            endpoint, connection_limit: opts[:max_connections]
+            endpoint, limit: opts[:max_connections]
           )
           @building_pipeline = false
         end

data/lib/3scale/backend/transactor/notify_batcher.rb

@@ -30,9 +30,13 @@ module ThreeScale
         end
 
         def notify(provider_key, usage)
-          # batch several notifications together so that we can process just one
+          # We need the master service ID to report its metrics. If it's not
+          # set, we don't need to notify anything.
+          # Batch several notifications together so that we can process just one
           # job for a group of them.
-          notify_batch(provider_key, usage)
+          unless configuration.master_service_id.to_s.empty?
+            notify_batch(provider_key, usage)
+          end
         end
 
         def notify_batch(provider_key, usage)

data/lib/3scale/backend/transactor/notify_job.rb

@@ -7,8 +7,6 @@ module ThreeScale
         extend Configurable
         @queue = :main
 
-        InvalidMasterServiceId = Class.new(ThreeScale::Backend::Error)
-
         class << self
           def perform_logged(provider_key, usage, timestamp, _enqueue_time)
             application_id = Application.load_id_by_key(master_service_id, provider_key)
@@ -16,12 +14,42 @@ module ThreeScale
             if application_id && Application.exists?(master_service_id, application_id)
               master_metrics = Metric.load_all(master_service_id)
 
-              ProcessJob.perform([{
-                service_id: master_service_id,
-                application_id: application_id,
-                timestamp: timestamp,
-                usage: master_metrics.process_usage(usage)
-              }])
+              begin
+                ProcessJob.perform([{
+                  service_id: master_service_id,
+                  application_id: application_id,
+                  timestamp: timestamp,
+                  usage: master_metrics.process_usage(usage)
+                }])
+              rescue MetricInvalid => e
+                # This happens when the master account in Porta does not have
+                # the notify metrics defined (by default "transactions" and
+                # "transactions/authorize"). These metrics need to be created in
+                # Porta, Apisonator does not have a way to guarantee that
+                # they're defined.
+                # Notice that this rescue prevents the job from being retried.
+                # Apisonator can't know when the metrics will be created (if
+                # ever) so it's better to log the error rather than retrying
+                # these jobs for an undefined period of time.
+                Worker.logger.notify(e)
+                return [false, "#{e}"]
+              rescue TransactionTimestampNotWithinRange => e
+                # This is very unlikely to happen. The timestamps in a notify
+                # job are not set by users, they are set by the listeners. If
+                # this error happens it might mean that:
+                # a) The worker started processing this job way after the
+                # listener produced it. This can happen for example if we make
+                # some requests to a listener with no workers. The listeners
+                # will enqueue some notify jobs. If we start a worker hours
+                # later, we might see this error.
+                # b) There's some kind of clock skew issue.
+                # c) There's a bug.
+                #
+                # We can't raise here, because then, the job will be retried,
+                # but it's going to fail always if it has an old timestamp.
+                Worker.logger.notify(e)
+                return [false, "#{provider_key} #{application_id} #{e}"]
+              end
             end
             [true, "#{provider_key} #{application_id || '--'}"]
           end
@@ -29,15 +57,7 @@ module ThreeScale
           private
 
           def master_service_id
-            value = configuration.master_service_id
-
-            unless value
-              raise InvalidMasterServiceId,
-                    "Can't find master service id. Make sure the \"master_service_id\" "\
-                    'configuration value is set correctly'
-            end
-
-            value.to_s
+            configuration.master_service_id.to_s
           end
         end
       end

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.0.0.pre1'
+    VERSION = '3.2.0'
   end
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>2.101.1</version>
+        <version>3.2.0</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -73,7 +73,7 @@
       </dependency>
           <dependency>
         <packageName>async-io</packageName>
-        <version>1.27.5</version>
+        <version>1.27.7</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -93,7 +93,7 @@
       </dependency>
           <dependency>
         <packageName>async-redis</packageName>
-        <version>0.4.1</version>
+        <version>0.5.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -371,7 +371,7 @@
       </dependency>
           <dependency>
         <packageName>json</packageName>
-        <version>2.1.0</version>
+        <version>2.3.1</version>
         <licenses>
                       <license>
               <name>ruby</name>
@@ -669,7 +669,7 @@
       </dependency>
           <dependency>
         <packageName>protocol-redis</packageName>
-        <version>0.2.0</version>
+        <version>0.5.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -719,7 +719,7 @@
       </dependency>
           <dependency>
         <packageName>rack</packageName>
-        <version>2.0.9</version>
+        <version>2.1.4</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre1
+  version: 3.2.0
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-19 00:00:00.000000000 Z
+date: 2021-01-19 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.