apisonator 2.100.0

data/lib/3scale/backend/transactor/notify_job.rb

@@ -0,0 +1,47 @@
+module ThreeScale
+  module Backend
+    module Transactor
+      #
+      # Job for notifying about backend calls.
+      class NotifyJob < BackgroundJob
+        extend Configurable
+        @queue = :main
+
+        InvalidMasterServiceId = Class.new(ThreeScale::Backend::Error)
+
+        class << self
+          def perform_logged(provider_key, usage, timestamp, _enqueue_time)
+            application_id = Application.load_id_by_key(master_service_id, provider_key)
+
+            if application_id && Application.exists?(master_service_id, application_id)
+              master_metrics = Metric.load_all(master_service_id)
+
+              ProcessJob.perform([{
+                service_id: master_service_id,
+                application_id: application_id,
+                timestamp: timestamp,
+                usage: master_metrics.process_usage(usage)
+              }])
+            end
+            [true, "#{provider_key} #{application_id || '--'}"]
+          end
+
+          private
+
+          def master_service_id
+            value = configuration.master_service_id
+
+            unless value
+              raise InvalidMasterServiceId,
+                    "Can't find master service id. Make sure the \"master_service_id\" "\
+                    'configuration value is set correctly'
+            end
+
+            value.to_s
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/3scale/backend/transactor/process_job.rb

@@ -0,0 +1,33 @@
+require '3scale/backend/transaction'
+
+module ThreeScale
+  module Backend
+    module Transactor
+      # Job for processing (aggregating and archiving) transactions.
+
+      ## WARNING: This is not a resque job, the .perform is called by another
+      ## job, either Report or NotifyJob it's meant to be like this in case we
+      ## want to detach it further
+      class ProcessJob
+
+        class << self
+          def perform(transactions)
+            transactions = preprocess(transactions)
+            Stats::Aggregator.process(transactions)
+          end
+
+          private
+
+          def preprocess(transactions)
+            transactions.map do |transaction_attrs|
+              transaction = Transaction.new(transaction_attrs)
+              transaction.ensure_on_time!
+              transaction
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/3scale/backend/transactor/report_job.rb

@@ -0,0 +1,84 @@
+module ThreeScale
+  module Backend
+    module Transactor
+
+      # Job for reporting transactions.
+      class ReportJob < BackgroundJob
+        @queue = :priority
+
+        class << self
+          def perform_logged(service_id, raw_transactions, _enqueue_time, context_info = {})
+            context_info ||= {} # avoid nils potentially existing in older versions
+            request_info = context_info['request'.freeze] || {}
+
+            transactions = parse_transactions(service_id, raw_transactions, request_info)
+            ProcessJob.perform(transactions) if !transactions.nil? && transactions.size > 0
+
+            # Last field was logs.size. Set it to 0 until we modify our parsers.
+            [true, "#{service_id} #{transactions.size} 0"]
+          rescue Error => error
+            ErrorStorage.store(service_id, error, context_info)
+            [false, "#{service_id} #{error}"]
+          rescue Exception => error
+            if error.class == ArgumentError &&
+                error.message == "invalid byte sequence in UTF-8"
+
+              ErrorStorage.store(service_id, NotValidData.new, context_info)
+              [false, "#{service_id} #{error}"]
+            else
+              raise error
+            end
+          end
+
+          private
+
+          def enqueue_time(args)
+            args[2]
+          end
+
+          def parse_transactions(service_id, raw_transactions, request_info)
+            transactions = []
+            exts = request_info['extensions'.freeze]&.symbolize_names || {}
+
+            group_by_application_id(service_id, raw_transactions) do |app_id, group|
+              group.each do |raw_transaction|
+
+                transaction = compose_transaction(service_id, app_id, raw_transaction, exts)
+                log         = raw_transaction['log']
+
+                transaction[:response_code] = log['code'] if transaction && log
+
+                transactions << transaction if transaction
+              end
+            end
+
+            transactions
+          end
+
+          def group_by_application_id(service_id, transactions, &block)
+            return [] if transactions.empty?
+            transactions = transactions.values if transactions.respond_to?(:values)
+            transactions.group_by do |transaction|
+              Application.extract_id!(service_id, transaction['app_id'], transaction['user_key'], transaction['access_token'])
+            end.each(&block)
+          end
+
+          def compose_transaction(service_id, app_id, raw_transaction, extensions)
+            usage = raw_transaction['usage']
+
+            if usage && !usage.empty?
+              metrics = Metric.load_all(service_id)
+              {
+                service_id:     service_id,
+                application_id: app_id,
+                timestamp:      raw_transaction['timestamp'],
+                usage:          metrics.process_usage(usage,
+                                                      extensions[:flat_usage] == '1'),
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/transactor/status.rb

@@ -0,0 +1,236 @@
+require 'json'
+
+module ThreeScale
+  module Backend
+    module Transactor
+      class Status
+        # This is the default field we respond with when using OAuth redirects
+        # We only use 'redirect_uri' if a request sent such a param. See #397.
+        REDIRECT_URI_FIELD = 'redirect_url'.freeze
+        private_constant :REDIRECT_URI_FIELD
+
+        def initialize(attributes)
+          @service_id      = attributes[:service_id]
+          @application     = attributes[:application]
+          @oauth           = attributes[:oauth]
+          @usage           = attributes[:usage]
+          @predicted_usage = attributes[:predicted_usage]
+          @values          = filter_values(attributes[:values] || {})
+          @timestamp       = attributes[:timestamp] || Time.now.getutc
+          @hierarchy_ext   = attributes[:hierarchy]
+          @flat_usage_ext  = attributes[:flat_usage]
+
+          raise 'service_id not specified' if @service_id.nil?
+          raise ':application is required' if @application.nil?
+
+          @redirect_uri_field = REDIRECT_URI_FIELD
+          @authorized  = true
+        end
+
+        attr_reader :service_id
+        attr_reader :application
+        attr_reader :oauth
+        attr_accessor :redirect_uri_field
+        attr_accessor :values
+
+        def flat_usage
+          @flat_usage_ext
+        end
+
+        def reject!(error)
+          @authorized = false
+          @rejection_reason_code ||= error.code
+          @rejection_reason_text ||= error.message
+        end
+
+        attr_reader :timestamp
+        attr_reader :rejection_reason_code
+        attr_reader :rejection_reason_text
+
+        # Returns the usage to be reported in an authrep request.
+        def usage
+          @predicted_usage ? nil : @usage
+        end
+
+        # Returns the predicted usage of an authorize request.
+        def predicted_usage
+          @predicted_usage ? @usage : nil
+        end
+
+        # Returns the actual usage. If there isn't one, returns the predicted
+        # usage. If there isn't an actual or predicted usage, returns nil.
+        def actual_or_predicted_usage
+          usage || predicted_usage
+        end
+
+        def authorized?
+          @authorized
+        end
+
+        def plan_name
+          @application.plan_name unless @application.nil?
+        end
+
+        def application_usage_reports
+          @usage_report ||= load_usage_reports @application
+        end
+
+        def value_for_usage_limit(usage_limit)
+          values = @values[usage_limit.period]
+          values && values[usage_limit.metric_id] || 0
+        end
+
+        def value_for_application_usage_limit(usage_limit)
+          values = @values[usage_limit.period]
+          values && values[usage_limit.metric_id] || 0
+        end
+
+        # provides a hierarchy hash with metrics as symbolic names
+        def hierarchy
+          @hierarchy ||= Metric.hierarchy service_id
+        end
+
+        def limit_headers(now = Time.now.utc)
+          # maybe filter by exceeded reports if not authorized
+          LimitHeaders.get(reports_to_calculate_limit_headers, now)
+        end
+
+        def to_xml(options = {})
+          xml = ''
+          xml << '<?xml version="1.0" encoding="UTF-8"?>'.freeze unless options[:skip_instruct]
+          xml << '<status>'.freeze
+
+          add_authorize_section(xml)
+
+          if oauth
+            add_application_section(xml)
+          end
+
+          hierarchy_reports = [] if @hierarchy_ext
+          if !@application.nil?
+            add_plan_section(xml, 'plan'.freeze, plan_name)
+            add_reports_section(xml, application_usage_reports)
+            hierarchy_reports.concat application_usage_reports if hierarchy_reports
+          end
+
+          if hierarchy_reports
+            add_hierarchy_section(xml, hierarchy_reports)
+          end
+
+          xml << '</status>'.freeze
+        end
+
+        private
+
+        # Returns the app usage reports needed to construct the limit headers.
+        # If the status does not have a 'usage', this method returns all the
+        # usage reports. Otherwise, it returns the reports associated with the
+        # metrics present in the 'usage' and their parents.
+        def reports_to_calculate_limit_headers
+          all_reports = application_usage_reports
+
+          return all_reports if (@usage.nil? || @usage.empty?)
+
+          metric_names_in_usage = @usage.keys
+          metrics = metric_names_in_usage | ascendants_names(metric_names_in_usage)
+          all_reports.select { |report| metrics.include?(report.metric_name) }
+        end
+
+        def ascendants_names(metric_names)
+          metric_names.flat_map do |metric_name|
+            Metric.ascendants(@service_id, metric_name)
+          end
+        end
+
+        # make sure the keys are Periods
+        def filter_values(values)
+          return nil if values.nil?
+          values.inject({}) do |acc, (k, v)|
+            key = begin
+                    Period[k]
+                  rescue Period::Unknown
+                    k
+                  end
+            acc[key] = v
+            acc
+          end
+        end
+
+        def add_hierarchy_section(xml, reports)
+          xml << '<hierarchy>'.freeze
+          with_report_and_hierarchy(reports) do |ur, children|
+            xml << '<metric name="'.freeze
+            xml << ur.metric_name << '" children="'.freeze
+            xml << (children ? children.join(' '.freeze) : '') << '"/>'.freeze
+          end
+          xml << '</hierarchy>'.freeze
+        end
+
+        # helper to iterate over reports and get relevant hierarchy info
+        def with_report_and_hierarchy(reports)
+          reports.each do |ur|
+            yield ur, hierarchy[ur.metric_name]
+          end
+        end
+
+        def add_plan_section(xml, tag, plan_name)
+          xml << "<#{tag}>"
+          xml << plan_name.to_s.encode(xml: :text) << "</#{tag}>"
+        end
+
+        def add_authorize_section(xml)
+          if authorized?
+            xml << '<authorized>true</authorized>'.freeze
+          else
+            xml << '<authorized>false</authorized><reason>'.freeze
+            xml << rejection_reason_text
+            xml << '</reason>'.freeze
+          end
+        end
+
+        def add_application_section(xml)
+          redirect_uri = application.redirect_url
+          xml << '<application>' \
+                 "<id>#{application.id}</id>" \
+                 "<key>#{application.keys.first}</key>" \
+                 "<#{@redirect_uri_field}>#{redirect_uri}</#{@redirect_uri_field}>" \
+                 '</application>'
+        end
+
+        def load_usage_reports(application)
+          # We might have usage limits that apply to metrics that no longer
+          # exist. In that case, the usage limit refers to a metric ID that no
+          # longer has a name associated to it. When that happens, we do not
+          # want to take into account that usage limit.
+          # This might happen, for example, when a Backend client decides to delete
+          # a metric and all the associated usage limits, but the operation fails
+          # for some of the usage limits.
+
+          reports = []
+          if !application.nil?
+            application.usage_limits.each do |usage_limit|
+              if application.metric_name(usage_limit.metric_id)
+                reports << UsageReport.new(self, usage_limit)
+              end
+            end
+          end
+          reports
+        end
+
+        def add_reports_section(xml, reports)
+          unless reports.empty?
+            xml_node = 'usage_reports>'.freeze
+            xml << '<'.freeze
+            xml << xml_node
+            reports.each do |report|
+              xml << report.to_xml
+            end
+            xml << '</'.freeze
+            xml << xml_node
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/3scale/backend/transactor/usage_report.rb

@@ -0,0 +1,182 @@
+module ThreeScale
+  module Backend
+    module Transactor
+      class Status
+        class UsageReport
+          attr_reader :type, :period
+
+          def initialize(status, usage_limit)
+            @status      = status
+            @usage_limit = usage_limit
+            @period      = usage_limit.period.new(status.timestamp)
+          end
+
+          def metric_name
+            @metric_name ||= @status.application.metric_name(metric_id)
+          end
+
+          def metric_id
+            @usage_limit.metric_id
+          end
+
+          def max_value
+            @usage_limit.value
+          end
+
+          def current_value
+            @current_value ||= @status.value_for_usage_limit(@usage_limit)
+          end
+
+          # Returns -1 if the period is eternity. Otherwise, returns the time
+          # remaining until the end of the period in seconds.
+          def remaining_time(from = Time.now)
+            if period.granularity == Period::Granularity::Eternity
+              -1
+            else
+              (period.finish - from).ceil
+            end
+          end
+
+          # Returns the number of identical calls that can be made before
+          # violating the limits defined in the usage report.
+          #
+          # Authrep (with actual usage): suppose that we have a metric with a
+          # daily limit of 10, a current usage of 0, and a given usage of 2.
+          # After taking into account the given usage, the number of identical
+          # calls that could be performed is (10-2)/2 = 4.
+          #
+          # Authorize (with predicted usage): suppose that we have a metric
+          # with a daily limit of 10, a current usage of 0, and a given usage
+          # of 2. This time, the given usage is not taken into account, as it
+          # is predicted, not to be reported. The number of identical calls
+          # that could be performed is 10/2 = 5.
+          #
+          # Returns -1 when there is not a limit in the number of calls.
+          def remaining_same_calls
+            return 0 if remaining <= 0
+
+            usage = compute_usage
+            usage > 0 ? remaining/usage : -1
+          end
+
+          def usage
+            @status.usage
+          end
+
+          def exceeded?
+            current_value > max_value
+          end
+
+          def authorized?
+            @status.authorized?
+          end
+
+          def inspect
+            "#<#{self.class.name} " \
+              "type=#{type} " \
+              "period=#{period} " \
+              "metric_name=#{metric_name} " \
+              "max_value=#{max_value} " \
+              "current_value=#{current_value}>"
+          end
+
+          def to_h
+            { period: period,
+              metric_name: metric_name,
+              max_value: max_value,
+              current_value: current_value }
+          end
+
+          def to_xml
+            xml = String.new
+            # Node header
+            add_head(xml)
+            # Node content
+            add_period(xml) if period != Period[:eternity]
+            add_values(xml)
+            # Node closing
+            add_tail(xml)
+            xml
+          end
+
+          private
+
+          def hierarchy
+            @status.hierarchy
+          end
+
+          def add_head(xml)
+            xml << '<usage_report metric="'.freeze
+            xml << metric_name.to_s << '" period="'.freeze
+            xml << period.to_s << '"'.freeze
+            xml << (exceeded? ? ' exceeded="true">'.freeze : '>'.freeze)
+          end
+
+          def add_period(xml)
+            xml << '<period_start>'.freeze
+            xml << period.start.strftime(TIME_FORMAT) << '</period_start>'.freeze
+            xml << '<period_end>'.freeze
+            xml << period.finish.strftime(TIME_FORMAT) << '</period_end>'.freeze
+          end
+
+          def add_values(xml)
+            xml << '<max_value>'.freeze
+            xml << max_value.to_s << '</max_value><current_value>'.freeze
+            xml << compute_current_value.to_s
+            xml << '</current_value>'
+          end
+
+          def add_tail(xml)
+            xml << '</usage_report>'.freeze
+          end
+
+          def remaining
+            # The remaining could be negative for several reasons:
+            # 1) We allow reports that do not check limits.
+            # 2) The reports included in authreps are async.
+            # 3) A usage passed by param in an authrep can go over the limits.
+            # However, a negative remaining does not make much sense. It's
+            # better to return just 0.
+            [max_value - compute_current_value, 0].max
+          end
+
+          def compute_usage
+            usage = @status.usage || @status.predicted_usage
+
+            return 0 unless usage
+
+            this_usage = usage[metric_name] || 0
+            res = Usage.get_from(this_usage)
+
+            add_descendants_usage(usage, res)
+          end
+
+          # helper to compute the current usage value after applying a possibly
+          # non-existent usage (or possibly unauthorized state)
+          def compute_current_value
+            # If not authorized or nothing to add, we just report the current
+            # value from the data store.
+            if authorized? && usage
+              this_usage = usage[metric_name] || 0
+              # this is an auth/authrep request and therefore we should sum the usage
+              computed_usage = Usage.get_from this_usage, current_value
+              # children can alter the resulting current value
+              add_descendants_usage(usage, computed_usage)
+            else
+              current_value
+            end
+          end
+
+          def add_descendants_usage(usages, parent_usage)
+            descendants = Metric.descendants(@status.service_id, metric_name)
+
+            descendants.reduce(parent_usage) do |acc, descendant|
+              descendant_usage = usages[descendant]
+              Usage.get_from descendant_usage, acc
+            end
+          end
+        end
+      end
+    end
+  end
+end