apisonator 3.0.1.1 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd4ac963d174350e0e93513a0bfe572110c9e90659f7907277996358c9761902
-  data.tar.gz: 3c327a805079b402cc65330cb4ff3e84a247ef9c57c781a9e4e103be283cc8a5
+  metadata.gz: edd1887f6aa71c17fb5279f98988c9a3b5e631e80bc1f4142b6619b5f5e24f48
+  data.tar.gz: b6478d14e88a177480396a7d46d5f4e33cea683aea17303175ad18ae646e546f
 SHA512:
-  metadata.gz: 52f2e4795a62a1ed314b21d1b3839dc78c2e6f8ee5bcb6fce02d9b5cd1ced3004915a78927b9fd667d210a2bb547fe34f27844dd267cdb405ff36eb629d7e1c8
-  data.tar.gz: df81a8aceb61a99c1151f3418c156ce70e1c18a9e2ab28db80d9f35424bf73fe82cfe46bed02c6da5e229cb212344432dd603520cd1dea8940e4448c14e6c3e8
+  metadata.gz: 944b75385817abc4d2f8828c61b2301d5cdfbb6c1bb681b7a3165590f23ad1b6c80451fcf579b8f3cf0010fe5a95e65605998f5d8ec94f1ff4ce690556b3011a
+  data.tar.gz: f3f44b1b2ebfe6c8fcbbf36b90dd44e5a7875d997a36b97288417472fa4156689508bfc95afdf196a7a168f768aa6e38521502b93078f128994959d184c1cd37

data/CHANGELOG.md

@@ -2,6 +2,26 @@
 
 Notable changes to Apisonator will be tracked in this document.
 
+## 3.1.0 - 2020-10-14
+
+### Added
+
+- Prometheus metrics for the internal API
+([#236](https://github.com/3scale/apisonator/pull/236)).
+- Docs with a detailed explanation about how counter updates are performed
+([#239](https://github.com/3scale/apisonator/pull/239)).
+
+### Changed
+
+- NotifyJobs are run only when the service ID is explicitly defined
+([#238](https://github.com/3scale/apisonator/pull/238)).
+
+### Fixed
+
+- Fixed corner case that raised "TransactionTimestampNotWithinRange" in notify
+jobs ([#235](https://github.com/3scale/apisonator/pull/235)).
+
+
 ## 3.0.1.1 - 2020-07-28
 
 ### Changed

data/Gemfile.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.1.1)
+    apisonator (3.1.0)
 
 GEM
   remote: https://rubygems.org/

data/Gemfile.on_prem.lock

@@ -35,7 +35,7 @@ GIT
 PATH
   remote: .
   specs:
-    apisonator (3.0.1.1)
+    apisonator (3.1.0)
 
 GEM
   remote: https://rubygems.org/

data/lib/3scale/backend/configuration.rb

@@ -1,6 +1,7 @@
 require '3scale/backend/configuration/loader'
 require '3scale/backend/environment'
 require '3scale/backend/configurable'
+require '3scale/backend/errors'
 
 module ThreeScale
   module Backend
@@ -77,9 +78,6 @@ module ThreeScale
       master_metrics = [:transactions, :transactions_authorize]
       config.master.metrics = Struct.new(*master_metrics).new
 
-      # Default config
-      config.master_service_id  = 1
-
       # This setting controls whether the listener can create event buckets in
       # Redis. We do not want all the listeners creating buckets yet, as we do
       # not know exactly the rate at which we can send events to Kinesis

data/lib/3scale/backend/listener_metrics.rb

@@ -4,14 +4,34 @@ require 'rack'
 module ThreeScale
   module Backend
     class ListenerMetrics
-      REQUEST_TYPES = {
+      AUTH_AND_REPORT_REQUEST_TYPES = {
         '/transactions/authorize.xml' => 'authorize',
         '/transactions/oauth_authorize.xml' => 'authorize_oauth',
         '/transactions/authrep.xml' => 'authrep',
         '/transactions/oauth_authrep.xml' => 'authrep_oauth',
         '/transactions.xml' => 'report'
       }
-      private_constant :REQUEST_TYPES
+      private_constant :AUTH_AND_REPORT_REQUEST_TYPES
+
+      INTERNAL_API_PATHS = [
+        [/\/services\/.*\/alert_limits/, 'alerts'],
+        [/\/services\/.*\/applications\/.*\/keys/, 'application_keys'],
+        [/\/services\/.*\/applications\/.*\/referrer_filters/, 'application_referrer_filters'],
+        [/\/services\/.*\/applications/, 'applications'],
+        [/\/services\/.*\/errors/, 'errors'],
+        [/\/events/, 'events'],
+        [/\/services\/.*\/metrics/, 'metrics'],
+        [/\/service_tokens/, 'service_tokens'],
+        [/\/services/, 'services'],
+        [/\/services\/.*\/stats/, 'stats'],
+        [/\/services\/.*\/plans\/.*\/usagelimits/, 'usage_limits'],
+        [/\/services\/.*\/applications\/.*\/utilization/, 'utilization'],
+      ].freeze
+      private_constant :INTERNAL_API_PATHS
+
+      # Most requests will be under 100ms, so use a higher granularity from there
+      TIME_BUCKETS = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+      private_constant :TIME_BUCKETS
 
       class << self
         ERRORS_4XX_TO_TRACK = Set[403, 404, 409].freeze
@@ -27,9 +47,12 @@ module ThreeScale
         end
 
         def report_resp_code(path, resp_code)
-          Yabeda.apisonator_listener.response_codes.increment(
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_codes.increment(
             {
-              request_type: REQUEST_TYPES[path],
+              request_type: req_type,
               resp_code: code_group(resp_code)
             },
             by: 1
@@ -37,8 +60,11 @@ module ThreeScale
         end
 
         def report_response_time(path, request_time)
-          Yabeda.apisonator_listener.response_times.measure(
-            { request_type: REQUEST_TYPES[path] },
+          req_type = req_type(path)
+          prometheus_group = prometheus_group(req_type)
+
+          Yabeda.send(prometheus_group).response_times.measure(
+            { request_type: req_type },
             request_time
           )
         end
@@ -69,8 +95,21 @@ module ThreeScale
                 comment 'Response times'
                 unit :seconds
                 tags %i[request_type]
-                # Most requests will be under 100ms, so use a higher granularity from there
-                buckets [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.25, 0.5, 0.75, 1]
+                buckets TIME_BUCKETS
+              end
+            end
+
+            group :apisonator_listener_internal_api do
+              counter :response_codes do
+                comment 'Response codes'
+                tags %i[request_type resp_code]
+              end
+
+              histogram :response_times do
+                comment 'Response times'
+                unit :seconds
+                tags %i[request_type]
+                buckets TIME_BUCKETS
               end
             end
           end
@@ -93,6 +132,24 @@ module ThreeScale
             'unknown'.freeze
           end
         end
+
+        def req_type(path)
+          AUTH_AND_REPORT_REQUEST_TYPES[path] || internal_api_req_type(path)
+        end
+
+        def internal_api_req_type(path)
+          (_regex, type) = INTERNAL_API_PATHS.find { |(regex, _)| regex.match path }
+          type
+        end
+
+        # Returns the group as defined in .define_metrics
+        def prometheus_group(request_type)
+          if AUTH_AND_REPORT_REQUEST_TYPES.values.include? request_type
+            :apisonator_listener
+          else
+            :apisonator_listener_internal_api
+          end
+        end
       end
     end
   end

data/lib/3scale/backend/transactor/notify_batcher.rb

@@ -30,9 +30,13 @@ module ThreeScale
         end
 
         def notify(provider_key, usage)
-          # batch several notifications together so that we can process just one
+          # We need the master service ID to report its metrics. If it's not
+          # set, we don't need to notify anything.
+          # Batch several notifications together so that we can process just one
           # job for a group of them.
-          notify_batch(provider_key, usage)
+          unless configuration.master_service_id.to_s.empty?
+            notify_batch(provider_key, usage)
+          end
         end
 
         def notify_batch(provider_key, usage)

data/lib/3scale/backend/transactor/notify_job.rb

@@ -7,8 +7,6 @@ module ThreeScale
         extend Configurable
         @queue = :main
 
-        InvalidMasterServiceId = Class.new(ThreeScale::Backend::Error)
-
         class << self
           def perform_logged(provider_key, usage, timestamp, _enqueue_time)
             application_id = Application.load_id_by_key(master_service_id, provider_key)
@@ -16,12 +14,42 @@ module ThreeScale
             if application_id && Application.exists?(master_service_id, application_id)
               master_metrics = Metric.load_all(master_service_id)
 
-              ProcessJob.perform([{
-                service_id: master_service_id,
-                application_id: application_id,
-                timestamp: timestamp,
-                usage: master_metrics.process_usage(usage)
-              }])
+              begin
+                ProcessJob.perform([{
+                  service_id: master_service_id,
+                  application_id: application_id,
+                  timestamp: timestamp,
+                  usage: master_metrics.process_usage(usage)
+                }])
+              rescue MetricInvalid => e
+                # This happens when the master account in Porta does not have
+                # the notify metrics defined (by default "transactions" and
+                # "transactions/authorize"). These metrics need to be created in
+                # Porta, Apisonator does not have a way to guarantee that
+                # they're defined.
+                # Notice that this rescue prevents the job from being retried.
+                # Apisonator can't know when the metrics will be created (if
+                # ever) so it's better to log the error rather than retrying
+                # these jobs for an undefined period of time.
+                Worker.logger.notify(e)
+                return [false, "#{e}"]
+              rescue TransactionTimestampNotWithinRange => e
+                # This is very unlikely to happen. The timestamps in a notify
+                # job are not set by users, they are set by the listeners. If
+                # this error happens it might mean that:
+                # a) The worker started processing this job way after the
+                # listener produced it. This can happen for example if we make
+                # some requests to a listener with no workers. The listeners
+                # will enqueue some notify jobs. If we start a worker hours
+                # later, we might see this error.
+                # b) There's some kind of clock skew issue.
+                # c) There's a bug.
+                #
+                # We can't raise here, because then, the job will be retried,
+                # but it's going to fail always if it has an old timestamp.
+                Worker.logger.notify(e)
+                return [false, "#{provider_key} #{application_id} #{e}"]
+              end
             end
             [true, "#{provider_key} #{application_id || '--'}"]
           end
@@ -29,15 +57,7 @@ module ThreeScale
           private
 
           def master_service_id
-            value = configuration.master_service_id
-
-            unless value
-              raise InvalidMasterServiceId,
-                    "Can't find master service id. Make sure the \"master_service_id\" "\
-                    'configuration value is set correctly'
-            end
-
-            value.to_s
+            configuration.master_service_id.to_s
           end
         end
       end

data/lib/3scale/backend/version.rb

@@ -1,5 +1,5 @@
 module ThreeScale
   module Backend
-    VERSION = '3.0.1.1'
+    VERSION = '3.1.0'
   end
 end

data/licenses.xml

@@ -23,7 +23,7 @@
       </dependency>
           <dependency>
         <packageName>apisonator</packageName>
-        <version>3.0.1.1</version>
+        <version>3.1.0</version>
         <licenses>
                       <license>
               <name>Apache 2.0</name>

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: apisonator
 version: !ruby/object:Gem::Version
-  version: 3.0.1.1
+  version: 3.1.0
 platform: ruby
 authors:
 - Adam Ciganek
@@ -16,7 +16,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-28 00:00:00.000000000 Z
+date: 2020-10-14 00:00:00.000000000 Z
 dependencies: []
 description: This gem provides a daemon that handles authorization and reporting of
   web services managed by 3scale.