apisonator 2.100.0

data/lib/3scale/backend/queue_storage.rb

@@ -0,0 +1,16 @@
+module ThreeScale
+  module Backend
+    class QueueStorage
+      def self.connection(env, cfg)
+        init_params = { url: cfg.queues && cfg.queues.master_name }
+        if %w(development test).include?(env)
+          init_params[:default_url] = '127.0.0.1:6379'
+        end
+        options = Backend::Storage::Helpers.config_with(cfg.queues,
+                                                        options: init_params)
+
+        Storage.new(options)
+      end
+    end
+  end
+end

data/lib/3scale/backend/rack.rb

@@ -0,0 +1,49 @@
+require '3scale/backend/configuration'
+require '3scale/backend/logging/middleware'
+require '3scale/backend/util'
+require '3scale/backend/rack/exception_catcher'
+require '3scale/backend/rack/prometheus'
+require '3scale/backend/rack/internal_error_catcher'
+require '3scale/backend'
+
+require 'rack'
+
+module ThreeScale
+  module Backend
+    module Rack
+      def self.run(rack)
+        rack.instance_eval do
+          use Rack::InternalErrorCatcher if Backend.production?
+
+          Backend::Logging::External.setup_rack self
+
+          if Backend.configuration.listener_prometheus_metrics.enabled
+            use Rack::Prometheus
+          end
+
+          loggers = Backend.configuration.request_loggers
+          log_writers = Backend::Logging::Middleware.writers loggers
+          use Backend::Logging::Middleware, writers: log_writers
+
+          map "/internal" do
+            require_relative "#{Backend::Util.root_dir}/app/api/api"
+
+            internal_api = Backend::API::Internal.new(
+              username: Backend.configuration.internal_api.user,
+              password: Backend.configuration.internal_api.password,
+              allow_insecure: !Backend.production?
+            )
+
+            use ::Rack::Auth::Basic do |username, password|
+              internal_api.helpers.check_password username, password
+            end if internal_api.helpers.credentials_set?
+
+            run internal_api
+          end
+
+          run Backend::Listener.new
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/rack/exception_catcher.rb

@@ -0,0 +1,136 @@
+require 'rack'
+require '3scale/backend/cors'
+
+module ThreeScale
+  module Backend
+    module Rack
+      class ExceptionCatcher
+        # These are the headers responded with when an error happens,
+        # and here we include the CORS ones.
+        # Note that this way of managing the errors is fundamentally
+        # broken, as important information gets lost. That is the case
+        # with response headers and other processing that has happened
+        # until an exception was raised.
+        # A refactoring to use Sinatra's error facilities is in order.
+        ERROR_HEADERS = {
+          'Content-Type'.freeze => 'application/vnd.3scale-v2.0+xml'.freeze,
+        }.merge(Backend::CORS.const_get(:HEADERS)).freeze
+        private_constant :ERROR_HEADERS
+
+        INVALID_BYTE_SEQUENCE_ERR_MSG = 'Invalid query parameters: '\
+          'invalid byte sequence in UTF-8'.freeze
+        private_constant :INVALID_BYTE_SEQUENCE_ERR_MSG
+
+        INVALID_PERCENT_ENCODING_ERR_MSG = 'Invalid query parameters: '\
+        'invalid %-encoding'.freeze
+        private_constant :INVALID_PERCENT_ENCODING_ERR_MSG
+
+        # Raised with invalid hash params such as:
+        # usage[]=1&usage[metric]=1.
+        # This is not the whole message. It contains the affected param at the end.
+        EXPECTED_HASH_ERR_MSG = 'Invalid query parameters: '\
+          'expected Hash (got Array)'.freeze
+        private_constant :EXPECTED_HASH_ERR_MSG
+
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          resp = @app.call(env)
+          filter_encoding_error_response resp, env
+        rescue Backend::Error => e
+          delete_sinatra_error! env
+          respond_with e.http_code, prepare_body(e.to_xml, env)
+        rescue Exception => e
+          unhandled_exception(e)
+        end
+
+        private
+
+        # raise up the chain an unhandled exception - but test first for an edge
+        # case with Rack key space limit for parsing requests.
+        # See https://github.com/rack/rack/blob/2.0.4/lib/rack/query_parser.rb#L166
+        def unhandled_exception(e)
+          if e.class == RangeError &&
+              e.message == 'exceeded available parameter key space'.freeze
+            respond_with 400, Backend::NotValidData.new.to_xml
+          else
+            raise e
+          end
+        end
+
+        # Private: Deletes 'sinatra.error' key in Rack's env hash.
+        # Some external services report 'sinatra.error' and we don't want it when
+        # the error is rescued and managed by us with the error handler.
+        #
+        # env - The environment Hash.
+        #
+        # Returns nothing.
+        def delete_sinatra_error!(env)
+          env['sinatra.error'.freeze] = nil
+        end
+
+        # Private: Prepares the body to include in the reponse.
+        #
+        # body - Proposed body String.
+        # env - The environment Hash.
+        #
+        # Returns String.
+        def prepare_body(body, env)
+          Backend::Listener.threescale_extensions(env)[:no_body] ? ''.freeze : body
+        end
+
+        # Returns Rack response.
+        #   Format https://rack.github.io/
+        #   Array with three elements:
+        #   *) The HTTP response code
+        #   *) A Hash of headers
+        #   *) The response body, which must respond to `each`
+        def respond_with(code, body)
+          [code, ERROR_HEADERS.dup, [body]]
+        end
+
+        # Private:
+        # Filter to transform response under specific conditions:
+        #   - http_status is 400 and error code refers to encoding issues.
+        #   - http_status is 400 and error code refers to a malformed hash param.
+        # When input request has one of those issues, Sinatra does not raise error
+        # and it prepares its own response. For backwards compatibility, we need
+        # to capture the error and customize response accordingly.
+        # Best and cleanest way to accomplish this is using error handlers. However,
+        # Sinatra 2.0.0 has a bug when registering error handlers and request has encoding issues.
+        # It has been reported in https://github.com/sinatra/sinatra/issues/1350
+        #
+        # resp - Rack response. Format https://rack.github.io/
+        #         Array with three elements:
+        #         *) The HTTP response code
+        #         *) A Hash of headers
+        #         *) The response body, which must respond to `each`
+        #
+        # env - The environment Hash.
+        #
+        # Returns Rack response.
+        def filter_encoding_error_response(resp, env)
+          return resp unless resp.first == 400
+          # According to http://www.rubydoc.info/github/rack/rack/master/file/SPEC#The_Body
+          # The Body must respond to each and must only yield String values.
+          resp_body = resp.last.inject('') { |acc, x| acc << x }
+
+          if resp_body == INVALID_BYTE_SEQUENCE_ERR_MSG
+            delete_sinatra_error! env
+            resp = respond_with 400, Backend::NotValidData.new.to_xml
+          elsif resp_body.start_with?(EXPECTED_HASH_ERR_MSG)
+            delete_sinatra_error! env
+            resp = respond_with 400, Backend::BadRequest.new.to_xml
+          elsif resp_body.start_with?(INVALID_PERCENT_ENCODING_ERR_MSG)
+            delete_sinatra_error! env
+            resp = respond_with 400, Backend::BadRequest.new(INVALID_PERCENT_ENCODING_ERR_MSG).to_xml
+          end
+
+          resp
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/rack/internal_error_catcher.rb

@@ -0,0 +1,23 @@
+module ThreeScale
+  module Backend
+    module Rack
+
+      # This middleware should be the last one to run. If there's an exception,
+      # instead of propagating it to the web server, we set our own error
+      # message. The reason is that each web server handles this differently.
+      # Puma returns a generic error message, while Falcon returns the message
+      # of the exception.
+      class InternalErrorCatcher
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          @app.call(env)
+        rescue
+          [500, {}, ["Internal Server Error\n".freeze]]
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/rack/prometheus.rb

@@ -0,0 +1,19 @@
+module ThreeScale
+  module Backend
+    module Rack
+      class Prometheus
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          began_at = Time.now.getutc
+          status, header, body = @app.call(env)
+          ListenerMetrics.report_resp_code(env['REQUEST_PATH'], status)
+          ListenerMetrics.report_response_time(env['REQUEST_PATH'], Time.now - began_at)
+          [status, header, body]
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/saas.rb

@@ -0,0 +1,6 @@
+if ThreeScale::Backend.configuration.saas
+  # SaaS-specific dependencies
+  require '3scale/backend/statsd'
+  require '3scale/backend/experiment'
+  require '3scale/backend/saas_analytics'
+end

data/lib/3scale/backend/saas_analytics.rb

@@ -0,0 +1,4 @@
+# Requires needed for the SaaS analytics system based on Kinesis + Redshift.
+require 'aws-sdk'
+require '3scale/backend/analytics/kinesis'
+require '3scale/backend/analytics/redshift'

data/lib/3scale/backend/server.rb

@@ -0,0 +1,30 @@
+require '3scale/backend/util'
+
+module ThreeScale
+  module Backend
+    module Server
+      def self.get(server_name)
+        server_file = server_name.tr('-', '_')
+        require "3scale/backend/server/#{server_file}"
+        server_class_name = server_file.tr('_', '').capitalize
+        ThreeScale::Backend::Server.const_get server_class_name
+      end
+
+      def self.list
+        Dir[File.join(ThreeScale::Backend::Util.root_dir, 'lib', '3scale', 'backend', 'server', '*.rb')].map do |s|
+          File.basename(s)[0..-4]
+        end
+      end
+
+      module Utils
+        def argv_add(argv, option, switch, *arguments)
+          if option
+            argv << switch
+            arguments.each { |a| argv << a }
+          end
+          argv
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/server/falcon.rb

@@ -0,0 +1,52 @@
+module ThreeScale
+  module Backend
+    module Server
+      class Falcon
+        extend ThreeScale::Backend::Server::Utils
+
+        def self.start(global_options, options, args)
+          # Falcon does not support:
+          # - options[:daemonize]
+          # - options[:logfile]
+          # - options[:errorfile]
+          # - options[:pidfile]
+
+          manifest = global_options[:manifest]
+          return unless manifest
+
+          argv = ['falcon']
+          argv_add argv, true, '--bind', 'http://0.0.0.0'
+          argv_add argv, options[:port], '--port', options[:port]
+
+          # Starts the prometheus server if needed. Just once even when spanning
+          # multiple workers.
+          argv_add argv, true, '--preload', 'lib/3scale/prometheus_server.rb'
+
+          server_model = manifest[:server_model]
+          argv_add argv, true, '--count', server_model[:workers].to_s
+        end
+
+        def self.restart(global_options, options, args)
+          argv = ['falcon', 'supervisor']
+          argv_add argv, true, 'restart'
+        end
+
+        def self.stop(global_options, options, args)
+          STDERR.puts 'Not implemented'
+        end
+
+        def self.status(global_options, options, args)
+          STDERR.puts 'Not implemented'
+        end
+
+        def self.stats(global_options, options, args)
+          STDERR.puts 'Not implemented'
+        end
+
+        def self.help(global_options, options, args)
+          system('falcon --help')
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/server/puma.rb

@@ -0,0 +1,71 @@
+module ThreeScale
+  module Backend
+    module Server
+      class Puma
+        extend ThreeScale::Backend::Server::Utils
+
+        CONFIG = 'config/puma.rb'
+        CONTROL_SOCKET = '3scale_backend.sock'
+        STATE = '3scale_backend.state'
+
+        EXPANDED_ROOT_PATH = File.expand_path(File.join(*Array.new(5, '..')),
+                                              __FILE__)
+
+        def self.socket_state_dir(env, default_dir)
+          if ['development', 'test'].include?(env) && File.writable?(default_dir)
+            default_dir
+          else
+            File.join('', 'tmp')
+          end
+        end
+
+        def self.start(global_options, options, args)
+          manifest = global_options[:manifest]
+          return unless manifest
+          argv = ['puma']
+          argv_add argv, options[:daemonize], '-d'
+          argv_add argv, options[:port], '-p', options[:port]
+          argv_add argv, options[:logfile], '--redirect-stdout', options[:logfile]
+          argv_add argv, options[:errorfile], '--redirect-stderr', options[:errorfile]
+          argv << '--redirect-append' if [options[:logfile], options[:errorfile]].any?
+          argv_add argv, options[:pidfile], '--pidfile', options[:pidfile]
+          # workaround Puma bug not phase-restarting correctly if no --dir is specified
+          argv_add argv, true, '--dir', global_options[:directory] ? global_options[:directory] : EXPANDED_ROOT_PATH
+          argv_add argv, true, '-C', CONFIG
+          ss_dir = socket_state_dir(global_options[:environment], global_options[:directory] || EXPANDED_ROOT_PATH)
+          argv_add argv, true, '-S', File.join(ss_dir, STATE)
+          argv_add argv, true, '--control', "unix://#{File.join(ss_dir, CONTROL_SOCKET)}"
+          server_model = manifest[:server_model]
+          argv_add argv, true, '-w', server_model[:workers].to_s
+          argv_add argv, true, '-t', "#{server_model[:min_threads]}:#{server_model[:max_threads]}"
+        end
+
+        def self.restart(global_options, options, args)
+          build_pumactl_cmdline(options[:'phased-restart'] ? 'phased-restart' : 'restart', global_options, options, args)
+        end
+
+        [:stop, :status, :stats].each do |cmd|
+          define_singleton_method cmd do |global_options, options, args|
+            build_pumactl_cmdline(__method__, global_options, options, args)
+          end
+        end
+
+        def self.help(global_options, options, args)
+          system('puma --help')
+        end
+
+        def self.build_pumactl_cmdline(cmd, global_options, options, args)
+          argv = ['pumactl']
+          ss_dir = socket_state_dir(global_options[:environment], global_options[:directory] || EXPANDED_ROOT_PATH)
+          argv_add argv, true, '-S', File.join(ss_dir, STATE)
+          argv_add argv, true, '-C', "unix://#{File.join(ss_dir, CONTROL_SOCKET)}"
+          argv_add argv, true, '-F', CONFIG
+          argv << cmd.to_s
+          argv
+        end
+        private_class_method :build_pumactl_cmdline
+
+      end
+    end
+  end
+end

data/lib/3scale/backend/service.rb

@@ -0,0 +1,317 @@
+module ThreeScale
+  module Backend
+    class Service
+      include Storable
+
+      # list of attributes to be fetched from storage
+      ATTRIBUTES = %i[state referrer_filters_required backend_version
+                      user_registration_required default_user_plan_id
+                      default_user_plan_name provider_key].freeze
+      private_constant :ATTRIBUTES
+
+      attr_reader :state
+      attr_accessor :provider_key, :id, :backend_version,
+        :default_user_plan_id, :default_user_plan_name
+      attr_writer :referrer_filters_required, :user_registration_required,
+        :default_service
+
+      class << self
+        include Memoizer::Decorator
+
+        def attribute_names
+          (ATTRIBUTES + %i[id default_service].freeze).freeze
+        end
+
+        # Returns true if a given service belongs to the provider with
+        # that key without loading the whole object.
+        #
+        def authenticate_service_id(service_id, provider_key)
+          provider_key == provider_key_for(service_id)
+        end
+        memoize :authenticate_service_id
+
+        def default_id(provider_key)
+          storage.get(storage_key_by_provider(provider_key, :id))
+        end
+        memoize :default_id
+
+        def default_id!(provider_key)
+          default_id(provider_key) or raise ProviderKeyInvalid, provider_key
+        end
+
+        def load_by_id(service_id)
+          return if service_id.nil?
+
+          service_attrs = get_service(id = service_id.to_s)
+          massage_service_attrs service_attrs
+
+          return if service_attrs[:provider_key].nil?
+
+          new(service_attrs.merge(id: id,
+            default_service: default_service?(service_attrs[:provider_key], id)
+          ))
+        end
+        memoize :load_by_id
+
+        def load_by_id!(service_id)
+          load_by_id(service_id) or raise ServiceIdInvalid, service_id
+        end
+
+        def load_with_provider_key!(id, provider_key)
+          id = Service.default_id(provider_key) if id.nil? || id.empty?
+          raise ProviderKeyInvalidOrServiceMissing, provider_key if id.nil? || id.empty?
+
+          service = Service.load_by_id(id.split('-').last) || Service.load_by_id!(id)
+
+          if service.provider_key != provider_key
+            # this is an error; let's raise in default_id! or raise invalid service
+            Service.default_id!(provider_key)
+            raise ServiceIdInvalid, id
+          end
+
+          service
+        end
+
+        def delete_by_id(service_id)
+          service = load_by_id!(service_id)
+
+          if service.default_service? and not service_is_the_only_one_for_provider(service_id)
+            raise ServiceIsDefaultService, service.id
+          end
+
+          service.delete_data
+          service.clear_cache
+        end
+
+        def exists?(service_id)
+          storage.exists(storage_key(service_id, 'provider_key'))
+        end
+
+        def get_service(id)
+          keys = ATTRIBUTES.map { |attr| storage_key(id, attr) }
+          values = storage.mget(keys)
+
+          result = {}
+          ATTRIBUTES.each_with_index do |key, idx|
+            result[key] = values[idx]
+          end
+          result
+        end
+
+        def list(provider_key)
+          storage.smembers(storage_key_by_provider(provider_key, :ids)) || []
+        end
+        memoize :list
+
+        def save!(attributes = {})
+          massage_set_user_registration_required attributes
+
+          new(attributes).save!
+        end
+
+        def storage_key(id, attribute)
+          encode_key("service/id:#{id}/#{attribute}")
+        end
+
+        def storage_key_by_provider(provider_key, attribute)
+          encode_key("service/provider_key:#{provider_key}/#{attribute}")
+        end
+
+        def clear_cache(provider_key, id)
+          provider_key_arg = [provider_key]
+          keys = Memoizer.build_keys_for_class(self,
+                    authenticate_service_id: [id, provider_key],
+                    default_id: provider_key_arg,
+                    load_by_id: [id],
+                    list: provider_key_arg,
+                    provider_key_for: [id])
+          Memoizer.clear keys
+        end
+
+        # Gets the provider key without loading the whole service
+        def provider_key_for(service_id)
+          storage.get(storage_key(service_id, 'provider_key'.freeze))
+        end
+        memoize :provider_key_for
+
+        private
+
+        def massage_service_attrs(service_attrs)
+          service_attrs[:referrer_filters_required] =
+            service_attrs[:referrer_filters_required].to_i > 0
+          service_attrs[:user_registration_required] =
+            massage_get_user_registration_required(
+              service_attrs[:user_registration_required])
+
+          service_attrs
+        end
+
+        # nil => true, 1 => true, '1' => true, 0 => false, '0' => false
+        def massage_get_user_registration_required(value)
+          value.nil? ? true : value.to_i > 0
+        end
+
+        def massage_set_user_registration_required(attributes)
+          if attributes[:user_registration_required].nil?
+            val = storage.get(storage_key(attributes[:id], :user_registration_required))
+            attributes[:user_registration_required] =
+              (!val.nil? && val.to_i == 0) ? false : true
+          end
+        end
+
+        def get_attr(id, attribute)
+          storage.get(storage_key(id, attribute))
+        end
+
+        def default_service?(provider_key, id)
+          default_id(provider_key) == id.to_s
+        end
+
+        def service_is_the_only_one_for_provider(service_id)
+          provider_key = provider_key_for(service_id)
+          services = list(provider_key)
+          services.size == 1 and services[0] == service_id.to_s
+        end
+      end
+
+      def initialize(attributes = {})
+        # :state is set as active in this method when:
+        # - The state key is not present in the attributes hash
+        # - The state key is present in the attributes hash but it has
+        #   the nil value
+        # This is done in order to not break compatibility for existing
+        # Services saved in the database, that do not contain the state
+        # key.
+        attributes[:state] ||= :active
+
+        super(attributes)
+      end
+
+      def default_service?
+        @default_service
+      end
+
+      def referrer_filters_required?
+        @referrer_filters_required
+      end
+
+      def user_registration_required?
+        @user_registration_required
+      end
+
+      def save!
+        set_as_default_if_needed
+        persist
+        clear_cache
+        self
+      end
+
+      def clear_cache
+        self.class.clear_cache(provider_key, id)
+      end
+
+      def storage_key(attribute)
+        self.class.storage_key id, attribute
+      end
+
+      def delete_data
+        delete_from_lists
+        delete_attributes
+        ErrorStorage.delete_all(id)
+      end
+
+      def to_hash
+        {
+          id: id,
+          state: state,
+          provider_key: provider_key,
+          backend_version: backend_version,
+          referrer_filters_required: referrer_filters_required?,
+          user_registration_required: user_registration_required?,
+          default_user_plan_id: default_user_plan_id,
+          default_user_plan_name: default_user_plan_name,
+          default_service: default_service?
+        }
+      end
+
+      def active?
+        state == :active
+      end
+      alias_method :active, :active?
+
+      def active=(value)
+        self.state = value ? :active : :suspended
+      end
+
+      private
+
+      def state=(value)
+        # only :active or nil will be considered as :active
+        # we assume nil is active because not having a state in an
+        # existing service means that is active in Services created before
+        # this change
+        @state = value.nil? || value.to_sym == :active ? :active : :suspended
+      end
+
+      def delete_attributes
+        keys = ATTRIBUTES.map { |attr| storage_key(attr) }
+        keys << storage_key_by_provider(:id) if default_service?
+        storage.del keys
+      end
+
+      def delete_from_lists
+        set = storage_key_by_provider :ids
+        storage.srem set, id
+        storage.srem encode_key('services_set'), id
+        storage.del set if default_service?
+      end
+
+      def storage_key_by_provider(attribute)
+        self.class.storage_key_by_provider provider_key, attribute
+      end
+
+      def set_as_default_if_needed
+        if @default_service.nil?
+          default_service_id = self.class.default_id(provider_key)
+          @default_service = default_service_id.nil?
+        end
+      end
+
+      def persist
+        persist_default(self.class.default_id(provider_key)) if default_service?
+        persist_attributes
+        persist_sets
+      end
+
+      def persist_default(old_default_id)
+        # we get all sorts of combinations of Strings and Fixnums here. Convert'em.
+        if old_default_id.to_i != id.to_i
+          storage.set storage_key_by_provider(:id), id
+          # we should now clear memoizations of the previous default service
+          self.class.clear_cache(provider_key, old_default_id)
+        end
+      end
+
+      def persist_attributes
+        persist_attribute :referrer_filters_required, referrer_filters_required? ? 1 : 0
+        persist_attribute :user_registration_required, user_registration_required? ? 1 : 0
+        persist_attribute :default_user_plan_id, default_user_plan_id, true
+        persist_attribute :default_user_plan_name, default_user_plan_name, true
+        persist_attribute :backend_version, backend_version, true
+        persist_attribute :provider_key, provider_key
+        persist_attribute :state, state.to_s if state
+      end
+
+      def persist_attribute(attribute, value, ignore_nils = false)
+        storage.set storage_key(attribute), value unless ignore_nils && value.nil?
+      end
+
+      def persist_sets
+        storage.sadd storage_key_by_provider(:ids), id
+        storage.sadd encode_key("services_set"), id
+        storage.sadd encode_key("provider_keys_set"), provider_key
+      end
+
+    end
+  end
+end