apisonator 3.0.0 → 3.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 455379ea79ee4377baa1060f0213d37d08c66135c6a0be9233536de917968303
-  data.tar.gz: d46f21858ba98539d51b0a32582e9b82c2ff6d08cbacb1b791d6cb9ea782024e
+  metadata.gz: 1f414f062cdabb59cdcc29b67a862a4400baa6e8d92ebeec909907ed881aa806
+  data.tar.gz: 2db24643307830c7cb2cef21da16a51aae8f9336b7bf35f90d4f22f82d15ff45
 SHA512:
-  metadata.gz: 5df4b993ffd0dd16cc9d75b4dc88652cba83d0767c967d10777191010ea87a8db4cf0b824adc3879e47e4221ff5a48cfdfe54954b76b4fb4537b8a514fe91caf
-  data.tar.gz: ee54b488232e0cf277d7bf052bbaab9edb96ba582b18859a7053a57a07ee50b9e932ed471e26a84cd60b9c7052c444b9595d07f98eb7d937eb83e4a5b28a1118
+  metadata.gz: 71ad48dc02408cab0de7f8264832489a08b935f55b47139a81ee7228cd71c783d8ebe81c7737c9b3035327535bd9b561784838486194e4b2ab808bb740fad753
+  data.tar.gz: 843744f70251b4d26365ab1fe2a4e7c73444b57629f3815c6043bbe467c70835050fdaabdd458e47323a11fb4294b01d1674dd5215f09b6647518adcfbf484da

data/CHANGELOG.md

@@ -2,6 +2,73 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.2.1 - 2021-01-22
+
+### Fixed
+
+- Reports of 0 hits no longer generate unnecessary stats keys in Redis
+([#247](https://github.com/3scale/apisonator/pull/247)).
+
+## 3.2.0 - 2021-01-19
+
+### Added
+
+- New endpoint in the internal API to get the provider key for a given (token,
+service_id) pair ([#243](https://github.com/3scale/apisonator/pull/243)).
+
+### Changed
+
+- The config file used when running in a Docker image now parses "1" and "true"
+(case-insensitive) as true
+([#245](https://github.com/3scale/apisonator/pull/245)).
+
+### Fixed
+
+- Fixed some metrics of the internal API that were not being counted
+correctly([#244](https://github.com/3scale/apisonator/pull/244)).
+
+
+## 3.1.0 - 2020-10-14
+
+### Added
+
+- Prometheus metrics for the internal API
+([#236](https://github.com/3scale/apisonator/pull/236)).
+- Docs with a detailed explanation about how counter updates are performed
+([#239](https://github.com/3scale/apisonator/pull/239)).
+
+### Changed
+
+- NotifyJobs are run only when the service ID is explicitly defined
+([#238](https://github.com/3scale/apisonator/pull/238)).
+
+### Fixed
+
+- Fixed corner case that raised "TransactionTimestampNotWithinRange" in notify
+jobs ([#235](https://github.com/3scale/apisonator/pull/235)).
+
+
+## 3.0.1.1 - 2020-07-28
+
+### Changed
+
+- Updated json gem to v2.3.1
+([#232](https://github.com/3scale/apisonator/pull/232)).
+
+## 3.0.1 - 2020-07-14
+
+### Fixed
+
+- The Prometheus counter of "5xx" errors has been fixed
+([#230](https://github.com/3scale/apisonator/pull/230)).
+
+### Changed
+
+- Gem updates: rack to v2.1.4
+([#228](https://github.com/3scale/apisonator/pull/228)) and async-redis to
+v0.5.0 ([#229](https://github.com/3scale/apisonator/pull/229)).
+
+
 ## 3.0.0 - 2020-06-19
 
 ### Removed

data/Gemfile.base

@@ -53,13 +53,13 @@ gem 'rake', '~> 13.0'
 gem 'builder', '= 3.2.3'
 # Use a patched resque to allow reusing their Airbrake Failure class
 gem 'resque', git: 'https://github.com/3scale/resque', branch: '3scale'
-gem 'rack', '~> 2.0.8'
+gem 'rack', '~> 2.1.4'
 gem 'sinatra', '~> 2.0.3'
 gem 'sinatra-contrib', '~> 2.0.3'
 # Optional external error logging services
 gem 'bugsnag', '~> 6', require: nil
 gem 'yabeda-prometheus', '~> 0.5.0'
-gem 'async-redis', '~> 0.4.1'
+gem 'async-redis', '~> 0.5'
 gem 'falcon', '~> 0.35'
 
 # Use a patched redis-rb that fixes an issue when trying to connect with

data/Gemfile.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.0)
+    apisonator (3.2.1)
 
 GEM
   remote: https://rubygems.org/
@@ -66,14 +66,15 @@ GEM
     async-http-cache (0.1.5)
       async-http
       protocol-http (~> 0.14)
-    async-io (1.27.5)
+    async-io (1.27.7)
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.4.1)
+    async-redis (0.5.0)
       async (~> 1.8)
       async-io (~> 1.10)
-      protocol-redis (~> 0.2.0)
+      async-pool (~> 0.2)
+      protocol-redis (~> 0.5.0)
     async-rspec (1.13.0)
       rspec (~> 3.0)
       rspec-files (~> 1.0)
@@ -118,7 +119,7 @@ GEM
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jmespath (1.3.1)
-    json (2.1.0)
+    json (2.3.1)
     license_finder (5.9.2)
       bundler
       rubyzip
@@ -161,7 +162,7 @@ GEM
     protocol-http2 (0.11.6)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
-    protocol-redis (0.2.0)
+    protocol-redis (0.5.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -171,7 +172,7 @@ GEM
     pry-doc (0.11.1)
       pry (~> 0.9)
       yard (~> 0.9)
-    rack (2.0.9)
+    rack (2.1.4)
     rack-protection (2.0.3)
       rack
     rack-test (0.8.2)
@@ -266,7 +267,7 @@ PLATFORMS
 DEPENDENCIES
   airbrake (= 4.3.1)
   apisonator!
-  async-redis (~> 0.4.1)
+  async-redis (~> 0.5)
   async-rspec
   aws-sdk (= 2.4.2)
   benchmark-ips (~> 2.7.2)
@@ -286,7 +287,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   pry-doc (~> 0.11.1)
   puma!
-  rack (~> 2.0.8)
+  rack (~> 2.1.4)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
   redis!

data/Gemfile.on_prem.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.0)
+    apisonator (3.2.1)
 
 GEM
   remote: https://rubygems.org/
@@ -63,14 +63,15 @@ GEM
     async-http-cache (0.1.5)
       async-http
       protocol-http (~> 0.14)
-    async-io (1.27.5)
+    async-io (1.27.7)
       async (~> 1.14)
     async-pool (0.2.0)
       async (~> 1.8)
-    async-redis (0.4.1)
+    async-redis (0.5.0)
       async (~> 1.8)
       async-io (~> 1.10)
-      protocol-redis (~> 0.2.0)
+      async-pool (~> 0.2)
+      protocol-redis (~> 0.5.0)
     async-rspec (1.13.0)
       rspec (~> 3.0)
       rspec-files (~> 1.0)
@@ -107,7 +108,7 @@ GEM
     hiredis (0.6.3)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    json (2.1.0)
+    json (2.3.1)
     license_finder (5.9.2)
       bundler
       rubyzip
@@ -149,7 +150,7 @@ GEM
     protocol-http2 (0.11.6)
       protocol-hpack (~> 1.4)
       protocol-http (~> 0.15)
-    protocol-redis (0.2.0)
+    protocol-redis (0.5.0)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
@@ -159,7 +160,7 @@ GEM
     pry-doc (0.11.1)
       pry (~> 0.9)
       yard (~> 0.9)
-    rack (2.0.9)
+    rack (2.1.4)
     rack-protection (2.0.3)
       rack
     rack-test (0.8.2)
@@ -249,7 +250,7 @@ PLATFORMS
 
 DEPENDENCIES
   apisonator!
-  async-redis (~> 0.4.1)
+  async-redis (~> 0.5)
   async-rspec
   benchmark-ips (~> 2.7.2)
   bugsnag (~> 6)
@@ -267,7 +268,7 @@ DEPENDENCIES
   pry-byebug (~> 3.5.1)
   pry-doc (~> 0.11.1)
   puma!
-  rack (~> 2.0.8)
+  rack (~> 2.1.4)
   rack-test (~> 0.8.2)
   rake (~> 13.0)
   redis!

data/app/api/internal/service_tokens.rb

@@ -7,6 +7,14 @@ module ThreeScale
           ServiceToken.exists?(token, service_id) ? 200 : 404
         end
 
+        get '/:token/:service_id/provider_key' do |token, service_id|
+          if ServiceToken.exists?(token, service_id)
+            { status: :found, provider_key: Service.provider_key_for(service_id) }.to_json
+          else
+            respond_with_404('token/service combination not found'.freeze)
+          end
+        end
+
         post '/' do
           check_tokens_param!
 

data/lib/3scale/backend/configuration.rb

@@ -1,6 +1,7 @@
 require '3scale/backend/configuration/loader'
 require '3scale/backend/environment'
 require '3scale/backend/configurable'
+require '3scale/backend/errors'
 
 module ThreeScale
   module Backend
@@ -77,9 +78,6 @@ module ThreeScale
       master_metrics = [:transactions, :transactions_authorize]
       config.master.metrics = Struct.new(*master_metrics).new
 
-      # Default config
-      config.master_service_id  = 1
-
       # This setting controls whether the listener can create event buckets in
       # Redis. We do not want all the listeners creating buckets yet, as we do
       # not know exactly the rate at which we can send events to Kinesis

data/lib/3scale/backend/listener_metrics.rb

@@ -4,14 +4,36 @@ require 'rack'
 module ThreeScale
   module Backend
     class ListenerMetrics
-      REQUEST_TYPES = {
+      AUTH_AND_REPORT_REQUEST_TYPES = {
         '/transactions/authorize.xml' => 'authorize',
         '/transactions/oauth_authorize.xml' => 'authorize_oauth',
         '/transactions/authrep.xml' => 'authrep',
         '/transactions/oauth_authrep.xml' => 'authrep_oauth',
         '/transactions.xml' => 'report'
       }
-      private_constant :REQUEST_TYPES
+      private_constant :AUTH_AND_REPORT_REQUEST_TYPES
+
+      # Only the first match is taken into account, that's why for example,
+      # "/\/services\/.*\/stats/" needs to appear before "/\/services/"
+      INTERNAL_API_PATHS = [
+        [/\/services\/.*\/alert_limits/, 'alerts'.freeze],
+        [/\/services\/.*\/applications\/.*\/keys/, 'application_keys'.freeze],
+        [/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'.freeze],
+        [/\/services\/.*\/applications\/.*\/utilization/, 'utilization'.freeze],
+        [/\/services\/.*\/applications/, 'applications'.freeze],
+        [/\/services\/.*\/errors/, 'errors'.freeze],
+        [/\/events/, 'events'.freeze],
+        [/\/services\/.*\/metrics/, 'metrics'.freeze],
+        [/\/service_tokens/, 'service_tokens'.freeze],
+        [/\/services\/.*\/stats/, 'stats'.freeze],
+        [/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'.freeze],
+        [/\/services/, 'services'.freeze],
+      ].freeze
+      private_constant :INTERNAL_API_PATHS
+
+      # Most requests will be under 100ms, so use a higher granularity from there
+      TIME_BUCKETS = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+      private_constant :TIME_BUCKETS
 
       class << self
         ERRORS_4XX_TO_TRACK = Set[403, 404, 409].freeze
@@ -27,9 +49,12 @@ module ThreeScale
         end
 
         def report_resp_code(path, resp_code)
-          Yabeda.apisonator_listener.response_codes.increment(
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_codes.increment(
             {
-              request_type: REQUEST_TYPES[path],
+              request_type: req_type,
               resp_code: code_group(resp_code)
             },
             by: 1
@@ -37,8 +62,11 @@ module ThreeScale
         end
 
         def report_response_time(path, request_time)
-          Yabeda.apisonator_listener.response_times.measure(
-            { request_type: REQUEST_TYPES[path] },
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_times.measure(
+            { request_type: req_type },
             request_time
           )
         end
@@ -69,8 +97,21 @@ module ThreeScale
                 comment 'Response times'
                 unit :seconds
                 tags %i[request_type]
-                # Most requests will be under 100ms, so use a higher granularity from there
-                buckets [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+                buckets TIME_BUCKETS
+              end
+            end
+
+            group :apisonator_listener_internal_api do
+              counter :response_codes do
+                comment 'Response codes'
+                tags %i[request_type resp_code]
+              end
+
+              histogram :response_times do
+                comment 'Response times'
+                unit :seconds
+                tags %i[request_type]
+                buckets TIME_BUCKETS
               end
             end
           end
@@ -93,6 +134,24 @@ module ThreeScale
             'unknown'.freeze
           end
         end
+
+        def req_type(path)
+          AUTH_AND_REPORT_REQUEST_TYPES[path] || internal_api_req_type(path)
+        end
+
+        def internal_api_req_type(path)
+          (_regex, type) = INTERNAL_API_PATHS.find { |(regex, _)| regex.match path }
+          type
+        end
+
+        # Returns the group as defined in .define_metrics
+        def prometheus_group(request_type)
+          if AUTH_AND_REPORT_REQUEST_TYPES.values.include? request_type
+            :apisonator_listener
+          else
+            :apisonator_listener_internal_api
+          end
+        end
       end
     end
   end

data/lib/3scale/backend/rack/prometheus.rb

@@ -8,7 +8,15 @@ module ThreeScale
 
         def call(env)
           began_at = Time.now.getutc
-          status, header, body = @app.call(env)
+
+          begin
+            status, header, body = @app.call(env)
+          rescue Exception => e
+            ListenerMetrics.report_resp_code(env['REQUEST_PATH'], 500)
+            ListenerMetrics.report_response_time(env['REQUEST_PATH'], Time.now - began_at)
+            raise e
+          end
+
           ListenerMetrics.report_resp_code(env['REQUEST_PATH'], status)
           ListenerMetrics.report_response_time(env['REQUEST_PATH'], Time.now - began_at)
           [status, header, body]

data/lib/3scale/backend/storage_async/client.rb

@@ -47,7 +47,7 @@ module ThreeScale
 
           endpoint = Async::IO::Endpoint.tcp(host, port)
           @redis_async = Async::Redis::Client.new(
-            endpoint, connection_limit: opts[:max_connections]
+            endpoint, limit: opts[:max_connections]
           )
           @building_pipeline = false
         end

data/lib/3scale/backend/transactor.rb

@@ -20,8 +20,14 @@ module ThreeScale
       def report(provider_key, service_id, transactions, context_info = {})
         service = Service.load_with_provider_key!(service_id, provider_key)
 
-        report_enqueue(service.id, transactions, context_info)
-        notify_report(provider_key, transactions.size)
+        # A usage of 0 does not affect rate-limits or stats, so we do not need
+        # to report it.
+        filtered_transactions = filter_usages_with_0(transactions.clone)
+
+        return if filtered_transactions.empty?
+
+        report_enqueue(service.id, filtered_transactions, context_info)
+        notify_report(provider_key, filtered_transactions.size)
       end
 
       def authorize(provider_key, params, context_info = {})
@@ -137,9 +143,17 @@ module ThreeScale
 
         usage = params[:usage]
 
-        if (usage || params[:log]) && status.authorized?
+        filtered_usage = filter_metrics_without_inc(usage.clone) if usage
+
+        if ((filtered_usage && !filtered_usage.empty?) || params[:log]) && status.authorized?
           application_id = status.application.id
-          report_enqueue(status.service_id, { 0 => {"app_id" => application_id, "usage" => usage, "log" => params[:log] } }, request: { extensions: request_info[:extensions] })
+
+          report_enqueue(
+            status.service_id,
+            { 0 => {"app_id" => application_id, "usage" => filtered_usage, "log" => params[:log] } },
+            request: { extensions: request_info[:extensions] }
+          )
+
           notify_authrep(provider_key, usage ? 1 : 0)
         else
           notify_authorize(provider_key)
@@ -182,6 +196,19 @@ module ThreeScale
         end
       end
 
+      def filter_usages_with_0(transactions)
+        # There are plenty of existing tests using both a string and a symbol
+        # when accessing the usage.
+        transactions.delete_if do |_idx, tx|
+          (usage = tx['usage'.freeze] || tx[:usage]) or next
+          filter_metrics_without_inc(usage).empty?
+        end
+      end
+
+      def filter_metrics_without_inc(usage)
+        usage.delete_if { |_metric, delta| delta.to_s == '0'.freeze }
+      end
+
       def storage
         Storage.instance
       end

data/lib/3scale/backend/transactor/notify_batcher.rb

@@ -30,9 +30,13 @@ module ThreeScale
         end
 
         def notify(provider_key, usage)
-          # batch several notifications together so that we can process just one
+          # We need the master service ID to report its metrics. If it's not
+          # set, we don't need to notify anything.
+          # Batch several notifications together so that we can process just one
           # job for a group of them.
-          notify_batch(provider_key, usage)
+          unless configuration.master_service_id.to_s.empty?
+            notify_batch(provider_key, usage)
+          end
         end
 
         def notify_batch(provider_key, usage)

data/lib/3scale/backend/transactor/notify_job.rb

@@ -7,8 +7,6 @@ module ThreeScale
         extend Configurable
         @queue = :main
 
-        InvalidMasterServiceId = Class.new(ThreeScale::Backend::Error)
-
         class << self
           def perform_logged(provider_key, usage, timestamp, _enqueue_time)
             application_id = Application.load_id_by_key(master_service_id, provider_key)
@@ -16,12 +14,42 @@ module ThreeScale
             if application_id && Application.exists?(master_service_id, application_id)
               master_metrics = Metric.load_all(master_service_id)
 
-              ProcessJob.perform([{
-                service_id: master_service_id,
-                application_id: application_id,
-                timestamp: timestamp,
-                usage: master_metrics.process_usage(usage)
-              }])
+              begin
+                ProcessJob.perform([{
+                  service_id: master_service_id,
+                  application_id: application_id,
+                  timestamp: timestamp,
+                  usage: master_metrics.process_usage(usage)
+                }])
+              rescue MetricInvalid => e
+                # This happens when the master account in Porta does not have
+                # the notify metrics defined (by default "transactions" and
+                # "transactions/authorize"). These metrics need to be created in
+                # Porta, Apisonator does not have a way to guarantee that
+                # they're defined.
+                # Notice that this rescue prevents the job from being retried.
+                # Apisonator can't know when the metrics will be created (if
+                # ever) so it's better to log the error rather than retrying
+                # these jobs for an undefined period of time.
+                Worker.logger.notify(e)
+                return [false, "#{e}"]
+              rescue TransactionTimestampNotWithinRange => e
+                # This is very unlikely to happen. The timestamps in a notify
+                # job are not set by users, they are set by the listeners. If
+                # this error happens it might mean that:
+                # a) The worker started processing this job way after the
+                # listener produced it. This can happen for example if we make
+                # some requests to a listener with no workers. The listeners
+                # will enqueue some notify jobs. If we start a worker hours
+                # later, we might see this error.
+                # b) There's some kind of clock skew issue.
+                # c) There's a bug.
+                #
+                # We can't raise here, because then, the job will be retried,
+                # but it's going to fail always if it has an old timestamp.
+                Worker.logger.notify(e)
+                return [false, "#{provider_key} #{application_id} #{e}"]
+              end
             end
             [true, "#{provider_key} #{application_id || '--'}"]
           end
@@ -29,15 +57,7 @@ module ThreeScale
           private
 
           def master_service_id
-            value = configuration.master_service_id
-
-            unless value
-              raise InvalidMasterServiceId,
-                    "Can't find master service id. Make sure the \"master_service_id\" "\
-                    'configuration value is set correctly'
-            end
-
-            value.to_s
+            configuration.master_service_id.to_s
           end
         end
       end

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.0.0'
+    VERSION = '3.2.1'
   end
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.0.0</version>
+        <version>3.2.1</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>
@@ -73,7 +73,7 @@
       </dependency>
           <dependency>
         <packageName>async-io</packageName>
-        <version>1.27.5</version>
+        <version>1.27.7</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -93,7 +93,7 @@
       </dependency>
           <dependency>
         <packageName>async-redis</packageName>
-        <version>0.4.1</version>
+        <version>0.5.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -371,7 +371,7 @@
       </dependency>
           <dependency>
         <packageName>json</packageName>
-        <version>2.1.0</version>
+        <version>2.3.1</version>
         <licenses>
                       <license>
               <name>ruby</name>
@@ -669,7 +669,7 @@
       </dependency>
           <dependency>
         <packageName>protocol-redis</packageName>
-        <version>0.2.0</version>
+        <version>0.5.0</version>
         <licenses>
                       <license>
               <name>MIT</name>
@@ -719,7 +719,7 @@
       </dependency>
           <dependency>
         <packageName>rack</packageName>
-        <version>2.0.9</version>
+        <version>2.1.4</version>
         <licenses>
                       <license>
               <name>MIT</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.2.1
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-19 00:00:00.000000000 Z
+date: 2021-01-22 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.