apisonator 2.100.0

data/lib/3scale/backend/storage_key_helpers.rb

@@ -0,0 +1,9 @@
+module ThreeScale
+  module Backend
+    module StorageKeyHelpers
+      def encode_key(key)
+        key.to_s.tr(' ', '+')
+      end
+    end
+  end
+end

data/lib/3scale/backend/storage_sync.rb

@@ -0,0 +1,43 @@
+# used to provide a Redis client based on a configuration object
+require 'uri'
+
+module ThreeScale
+  module Backend
+    class StorageSync
+      include Configurable
+
+      class << self
+        # Returns a shared instance of the storage. If there is no instance yet,
+        # creates one first. If you want to always create a fresh instance, set
+        # the +reset+ parameter to true.
+        def instance(reset = false)
+          if reset || @instance.nil?
+            @instance = new(Storage::Helpers.config_with(configuration.redis,
+                            options: get_options))
+          else
+            @instance
+          end
+        end
+
+        def new(options)
+          Redis.new options
+        end
+
+        private
+
+        if ThreeScale::Backend.production?
+          def get_options
+            {}
+          end
+        else
+          DEFAULT_SERVER = '127.0.0.1:22121'.freeze
+          private_constant :DEFAULT_SERVER
+
+          def get_options
+            { default_url: DEFAULT_SERVER }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/transaction.rb

@@ -0,0 +1,62 @@
+module ThreeScale
+  module Backend
+    class Transaction
+      # We accept transactions with a timestamp ts where ts:
+      # now - REPORT_DEADLINE_PAST <= ts <= now + REPORT_DEADLINE_FUTURE
+      REPORT_DEADLINE_PAST = 24*60*60
+      private_constant :REPORT_DEADLINE_PAST
+
+      REPORT_DEADLINE_FUTURE = 60*60
+      private_constant :REPORT_DEADLINE_FUTURE
+
+      # We can define an allowed range assuming Time.now = 0
+      DEADLINE_RANGE = (-REPORT_DEADLINE_PAST..REPORT_DEADLINE_FUTURE).freeze
+      private_constant :DEADLINE_RANGE
+
+      ATTRIBUTES = [:service_id, :application_id, :timestamp,
+                    :log, :usage, :response_code]
+      private_constant :ATTRIBUTES
+
+      class_eval { attr_accessor *ATTRIBUTES }
+
+      def initialize(params = {})
+        ATTRIBUTES.each { |attr| send("#{attr}=", (params[attr] || params[attr.to_s])) }
+      end
+
+      def timestamp=(value = nil)
+        if value.is_a?(Time)
+          @timestamp = value
+        else
+          @timestamp = Time.parse_to_utc(value) || Time.now.getutc
+        end
+      end
+
+
+      def extract_response_code
+        if (response_code.is_a?(String) && response_code =~ /\A\d{3}\z/) ||
+           (response_code.is_a?(Integer) && (100 ..999).cover?(response_code) )
+          response_code.to_i
+        else
+          false
+        end
+      end
+
+      # Validates if transaction timestamp is within accepted range
+      #
+      # @return [unspecified] if the timestamp is within the valid range.
+      # @raise [TransactionTimestampTooOld] if the timestamp is too old
+      # @raise [TransactionTimestampTooNew] if the timestamp is too new
+      def ensure_on_time!
+        time_diff_sec = timestamp.to_i - Time.now.to_i
+
+        unless DEADLINE_RANGE.cover?(time_diff_sec)
+          if time_diff_sec < 0
+            fail(TransactionTimestampTooOld, REPORT_DEADLINE_PAST)
+          else
+            fail(TransactionTimestampTooNew, REPORT_DEADLINE_FUTURE)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/transactor.rb

@@ -0,0 +1,177 @@
+require '3scale/backend/transactor/notify_batcher'
+require '3scale/backend/transactor/notify_job'
+require '3scale/backend/transactor/process_job'
+require '3scale/backend/transactor/report_job'
+require '3scale/backend/transactor/usage_report'
+require '3scale/backend/transactor/status'
+require '3scale/backend/transactor/limit_headers'
+require '3scale/backend/errors'
+require '3scale/backend/validators'
+require '3scale/backend/stats/keys'
+
+module ThreeScale
+  module Backend
+    # Methods for reporting and authorizing transactions.
+    module Transactor
+      include Backend::StorageKeyHelpers
+      include NotifyBatcher
+      extend self
+
+      def report(provider_key, service_id, transactions, context_info = {})
+        service = Service.load_with_provider_key!(service_id, provider_key)
+
+        report_enqueue(service.id, transactions, context_info)
+        notify_report(provider_key, transactions.size)
+      end
+
+      def authorize(provider_key, params, context_info = {})
+        do_authorize :authorize, provider_key, params, context_info
+      end
+
+      def oauth_authorize(provider_key, params, context_info = {})
+        do_authorize :oauth_authorize, provider_key, params, context_info
+      end
+
+      def authrep(provider_key, params, context_info = {})
+        do_authrep :authrep, provider_key, params, context_info
+      end
+
+      def oauth_authrep(provider_key, params, context_info = {})
+        do_authrep :oauth_authrep, provider_key, params, context_info
+      end
+
+      def utilization(service_id, application_id)
+        application = Application.load!(service_id, application_id)
+        application.load_metric_names
+        usage = Usage.application_usage(application, Time.now.getutc)
+        status = Status.new(service_id: service_id,
+                            application: application,
+                            values: usage)
+        Validators::Limits.apply(status, {})
+        status.application_usage_reports
+      end
+
+      private
+
+      def validate(oauth, provider_key, report_usage, params, request_info)
+        service = Service.load_with_provider_key!(params[:service_id], provider_key)
+        # service_id cannot be taken from params since it might be missing there
+        service_id = service.id
+
+        app_id = params[:app_id]
+        # TODO: make sure params are nil if they are empty up the call stack
+        # Note: app_key is an exception, as it being empty is semantically
+        # significant.
+        params[:app_id] = nil if app_id && app_id.empty?
+
+        if oauth
+          if app_id.nil?
+            access_token = params[:access_token]
+            access_token = nil if access_token && access_token.empty?
+
+            if access_token.nil?
+              raise ApplicationNotFound.new nil if app_id.nil?
+            else
+              app_id = get_token_ids(access_token, service_id, app_id)
+              # update params, since they are checked elsewhere
+              params[:app_id] = app_id
+            end
+          end
+
+          validators = Validators::OAUTH_VALIDATORS
+        else
+          validators = Validators::VALIDATORS
+        end
+
+        params[:user_key] = nil if params[:user_key] && params[:user_key].empty?
+        application = Application.load_by_id_or_user_key!(service_id,
+                                                          app_id,
+                                                          params[:user_key])
+        now          = Time.now.getutc
+        usage_values = Usage.application_usage(application, now)
+        extensions   = request_info && request_info[:extensions] || {}
+        status_attrs = {
+          service_id:      service_id,
+          application:     application,
+          oauth:           oauth,
+          usage:           params[:usage],
+          predicted_usage: !report_usage,
+          values:          usage_values,
+          # hierarchy parameter adds information in the response needed
+          # to derive which limits affect directly or indirectly the
+          # metrics for which authorization is requested.
+          hierarchy:       extensions[:hierarchy] == '1',
+          flat_usage:      extensions[:flat_usage] == '1'
+        }
+
+        application.load_metric_names
+
+        # returns a status object
+        apply_validators(validators, status_attrs, params)
+      end
+
+      def get_token_ids(token, service_id, app_id)
+        begin
+          token_aid = OAuth::Token::Storage.get_credentials(token, service_id)
+        rescue AccessTokenInvalid => e
+          # Yep, well, er. Someone specified that it is OK to have an
+          # invalid token if an app_id is specified. Somehow passing in
+          # a user_key is still not enough, though...
+          raise e if app_id.nil?
+        end
+
+        # We only take the token ids into account if we had no parameter ids
+        if app_id.nil?
+          app_id = token_aid
+        end
+
+        app_id
+      end
+
+      def do_authorize(method, provider_key, params, context_info)
+        notify_authorize(provider_key)
+        validate(method == :oauth_authorize, provider_key, false, params, context_info[:request])
+      end
+
+      def do_authrep(method, provider_key, params, context_info)
+        request_info = context_info[:request] || {}
+        status = begin
+                   validate(method == :oauth_authrep, provider_key, true, params, request_info)
+                 rescue ApplicationNotFound => e
+                   # we still want to track these
+                   notify_authorize(provider_key)
+                   raise e
+                 end
+
+        usage = params[:usage]
+
+        if (usage || params[:log]) && status.authorized?
+          application_id = status.application.id
+          report_enqueue(status.service_id, { 0 => {"app_id" => application_id, "usage" => usage, "log" => params[:log] } }, request: { extensions: request_info[:extensions] })
+          notify_authrep(provider_key, usage ? 1 : 0)
+        else
+          notify_authorize(provider_key)
+        end
+
+        status
+      end
+
+      # This method applies the validators in the given order. If there is one
+      # that fails, it stops there instead of applying all of them.
+      # Returns a Status instance.
+      def apply_validators(validators, status_attrs, params)
+        status = Status.new(status_attrs)
+        validators.all? { |validator| validator.apply(status, params) }
+        status
+      end
+
+      def report_enqueue(service_id, data, context_info)
+        Resque.enqueue(ReportJob, service_id, data, Time.now.getutc.to_f, context_info)
+      end
+
+      def storage
+        Storage.instance
+      end
+    end
+  end
+end

data/lib/3scale/backend/transactor/limit_headers.rb

@@ -0,0 +1,54 @@
+module ThreeScale
+  module Backend
+    module Transactor
+      module LimitHeaders
+        class << self
+          # Note: -1 is used in the remaining of usage reports to indicate that
+          # there is no limit.
+
+          UNCONSTRAINED = { remaining: -1, reset: -1 }.freeze
+          private_constant :UNCONSTRAINED
+
+          def get(reports, now = Time.now.utc)
+            report = most_constrained_report reports
+            if report && report.remaining_same_calls != -1
+              {
+                remaining: report.remaining_same_calls,
+                reset: report.remaining_time(now),
+                'max-value': report.max_value
+              }
+            else
+              UNCONSTRAINED
+            end
+          end
+
+          private
+
+          # Will return the most constrained report
+          def most_constrained_report(reports)
+            reports.min do |a, b|
+              compare a, b
+            end
+          end
+
+          # Warning: comparison here returns from most to least constrained
+          # usage report (ie. in descending order)
+          def compare(one, other)
+            other_rem = other.remaining_same_calls
+            one_rem = one.remaining_same_calls
+            if one_rem == other_rem
+              # Note: the line below is correctly reversing the operands
+              # This is done to have the larger period ordered before the
+              # shorter ones when the remaining hits are the same.
+              other.period.granularity <=> one.period.granularity
+            elsif other_rem == -1 || one_rem < other_rem
+              -1
+            else
+              1
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/transactor/notify_batcher.rb

@@ -0,0 +1,139 @@
+require 'resque'
+require '3scale/backend/configuration'
+
+module ThreeScale
+  module Backend
+    module Transactor
+
+      # This module is responsible for scheduling Notify jobs. These jobs are
+      # used to report the usage of some metrics specified in the master
+      # account.
+      module NotifyBatcher
+        include Resque::Helpers
+        include Configurable
+
+        def notify_authorize(provider_key)
+          notify(provider_key, configuration.master.metrics.transactions_authorize => 1)
+        end
+
+        def notify_authrep(provider_key, transactions)
+          notify(provider_key, configuration.master.metrics.transactions_authorize => 1,
+                 configuration.master.metrics.transactions => transactions)
+        end
+
+        def notify_report(provider_key, transactions)
+          notify(provider_key, configuration.master.metrics.transactions => transactions)
+        end
+
+        def key_for_notifications_batch
+          "notify/aggregator/batch"
+        end
+
+        def notify(provider_key, usage)
+          # batch several notifications together so that we can process just one
+          # job for a group of them.
+          notify_batch(provider_key, usage)
+        end
+
+        def notify_batch(provider_key, usage)
+          # discard seconds so that all the notifications are stored in the same
+          # bucket, because aggregation is done at the minute level.
+          tt = Time.now.getutc
+          tt = tt - tt.sec
+
+          encoded = Yajl::Encoder.encode({
+            provider_key: provider_key,
+            usage: usage,
+            time: tt.to_s
+          })
+
+          num_elements = storage.rpush(key_for_notifications_batch, encoded)
+
+          if (num_elements  % configuration.notification_batch) == 0
+            # batch is full
+            process_batch(num_elements)
+          end
+        end
+
+        def get_batch(num_elements)
+          storage.pipelined do
+            storage.lrange(key_for_notifications_batch, 0, num_elements - 1)
+            storage.ltrim(key_for_notifications_batch, num_elements, -1)
+          end.first
+        end
+
+        def process_batch(num_elements)
+          do_batch(get_batch num_elements)
+        end
+
+        def do_batch(list)
+          all = Hash.new
+
+          list.each do |item|
+            obj = decode(item)
+
+            provider_key = obj['provider_key'.freeze]
+            time = obj['time'.freeze]
+            usage = obj['usage'.freeze]
+
+            if usage.nil?
+              obj['usage'.freeze] = {}
+            end
+
+            bucket_key = "#{provider_key}-" << time
+            bucket_obj = all[bucket_key]
+
+            if bucket_obj.nil?
+              all[bucket_key] = obj
+            else
+              bucket_usage = bucket_obj['usage'.freeze]
+
+              usage.each do |metric_name, value|
+                bucket_usage[metric_name] =
+                  bucket_usage.fetch(metric_name, 0) + value.to_i
+              end
+            end
+          end
+
+          enqueue_ts = Time.now.utc.to_f
+
+          all.each do |_, v|
+            enqueue_notify_job(v['provider_key'.freeze],
+                               v['usage'.freeze],
+                               v['time'.freeze],
+                               enqueue_ts)
+          end
+        end
+
+        private
+
+        def enqueue_notify_job(provider_key, usage, timestamp, enqueue_ts)
+          ::Resque.enqueue(NotifyJob,
+                           provider_key,
+                           usage,
+                           timestamp,
+                           enqueue_ts)
+        end
+
+        if ThreeScale::Backend.test?
+          module Test
+            def get_full_batch
+              storage.pipelined do
+                storage.lrange(key_for_notifications_batch, 0, -1)
+                storage.del(key_for_notifications_batch)
+              end.first
+            end
+
+            def process_full_batch
+              do_batch(get_full_batch)
+            end
+          end
+
+          private_constant :Test
+
+          include Test
+        end
+      end
+    end
+  end
+end