apisonator 2.100.0

data/lib/3scale/backend/background_job.rb

@@ -0,0 +1,65 @@
+module ThreeScale
+  module Backend
+
+    class BackgroundJob
+      include Configurable
+
+      EMPTY_HOOKS = [].freeze
+      Error = Class.new StandardError
+
+      class << self
+        def perform(*args)
+          perform_wrapper(args)
+        end
+
+        def perform_logged(*_args)
+          raise "This should be overloaded."
+        end
+
+        # Disable hooks to improve performance. Profiling tests show that some
+        # significant CPU resources were consumed sorting and filtering these.
+        def hooks
+          EMPTY_HOOKS
+        end
+
+        private
+
+        def enqueue_time(args)
+          args.last or raise('Enqueue time not specified')
+        end
+
+        def perform_wrapper(args)
+          start_time = Time.now.getutc
+          status_ok, message = perform_logged(*args)
+          stats_mem = Memoizer.stats
+          end_time = Time.now.getutc
+
+          raise Error, 'No job message given' unless message
+          prefix = log_class_name + ' ' + message
+
+          if status_ok
+            Worker.logger.info(prefix +
+              " #{(end_time - start_time).round(5)}" +
+              " #{(end_time.to_f - enqueue_time(args)).round(5)}"+
+              " #{stats_mem[:size]} #{stats_mem[:count]} #{stats_mem[:hits]}")
+
+            if configuration.worker_prometheus_metrics.enabled
+              update_prometheus_metrics(end_time - start_time)
+            end
+          else
+            Worker.logger.error("#{log_class_name} " + message)
+          end
+        end
+
+        def log_class_name
+          self.name.split('::').last
+        end
+
+        def update_prometheus_metrics(runtime)
+          WorkerMetrics.increase_job_count(log_class_name)
+          WorkerMetrics.report_runtime(log_class_name, runtime)
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/configurable.rb

@@ -0,0 +1,20 @@
+require '3scale/backend/configuration'
+
+module ThreeScale
+  module Backend
+    # Include this into any class to provide convenient access to the configuration.
+    module Configurable
+      def self.included(base)
+        base.extend(self)
+      end
+
+      def configuration
+        ThreeScale::Backend.configuration
+      end
+
+      def configuration=(cfg)
+        ThreeScale::Backend.configuration=(cfg)
+      end
+    end
+  end
+end

data/lib/3scale/backend/configuration.rb

@@ -0,0 +1,151 @@
+require '3scale/backend/configuration/loader'
+require '3scale/backend/environment'
+require '3scale/backend/configurable'
+
+module ThreeScale
+  module Backend
+    class << self
+      attr_accessor :configuration
+
+      def configure
+        yield configuration
+      end
+
+      private
+
+      def parse_int(value, default)
+        case value
+        when "", nil, false then default
+        else Integer(value)
+        end
+      end
+    end
+
+    NOTIFICATION_BATCH_DEFAULT = 10000
+    private_constant :NOTIFICATION_BATCH_DEFAULT
+
+    CONFIG_MASTER_METRICS_TRANSACTIONS_DEFAULT = "transactions".freeze
+    private_constant :CONFIG_MASTER_METRICS_TRANSACTIONS_DEFAULT
+    CONFIG_MASTER_METRICS_TRANSACTIONS_AUTHORIZE_DEFAULT = "transactions/authorize".freeze
+    private_constant :CONFIG_MASTER_METRICS_TRANSACTIONS_AUTHORIZE_DEFAULT
+
+    CONFIG_DELETE_STATS_BATCH_SIZE = 50
+    private_constant :CONFIG_DELETE_STATS_BATCH_SIZE
+    CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE = 1000
+    private_constant :CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE
+
+    @configuration = Configuration::Loader.new
+
+    # assign @configuration first, since code can depend on the attr_reader
+    @configuration.tap do |config|
+      # To distinguish between SaaS and on-premises mode.
+      config.saas = true
+
+      config.request_loggers = [:text]
+      config.workers_logger_formatter = :text
+
+      # Add configuration sections
+      config.add_section(:queues, :master_name, :sentinels, :role,
+                         :connect_timeout, :read_timeout, :write_timeout)
+      config.add_section(:redis, :url, :proxy, :sentinels, :role,
+                         :connect_timeout, :read_timeout, :write_timeout,
+                         :async)
+      config.add_section(:analytics_redis, :server,
+                         :connect_timeout, :read_timeout, :write_timeout)
+      config.add_section(:hoptoad, :service, :api_key)
+      config.add_section(:stats, :bucket_size, :delete_batch_size, :delete_partition_batch_size)
+      config.add_section(:redshift, :host, :port, :dbname, :user, :password)
+      config.add_section(:statsd, :host, :port)
+      config.add_section(:internal_api, :user, :password)
+      config.add_section(:oauth, :max_token_size)
+      config.add_section(:master, :metrics)
+      config.add_section(:worker_prometheus_metrics, :enabled, :port)
+      config.add_section(:listener_prometheus_metrics, :enabled, :port)
+
+      config.add_section(
+          :async_worker,
+
+          # Max number of jobs in the reactor
+          :max_concurrent_jobs,
+          # Max number of jobs in memory pending to be added to the reactor
+          :max_pending_jobs,
+          # Seconds to wait before fetching more jobs when the number of jobs
+          # in memory has reached max_pending_jobs.
+          :seconds_before_fetching_more
+      )
+
+      # Configure nested fields
+      master_metrics = [:transactions, :transactions_authorize]
+      config.master.metrics = Struct.new(*master_metrics).new
+
+      # Default config
+      config.master_service_id  = 1
+
+      # This setting controls whether the listener can create event buckets in
+      # Redis. We do not want all the listeners creating buckets yet, as we do
+      # not know exactly the rate at which we can send events to Kinesis
+      # without problems.
+      # By default, we will allow creating buckets in any environment that is
+      # not 'production'.
+      # Notice that in order to create buckets, you also need to execute this
+      # rake task: stats:buckets:enable
+      config.can_create_event_buckets = !production?
+
+      config.legacy_referrer_filters = false
+
+      # Load configuration from a file.
+      config.load!([
+        '/etc/3scale_backend.conf',
+        '~/.3scale_backend.conf',
+        ENV['CONFIG_FILE']
+      ].compact)
+
+      ## this means that there will be a NotifyJob for every X notifications (this is
+      ## the call to master)
+      config.notification_batch = parse_int(config.notification_batch,
+                                            NOTIFICATION_BATCH_DEFAULT)
+
+      # Assign default values to some configuration values
+      # that might been set in the config file but their
+      # environment variables not, or not have been set
+			# but should always have at least a default value.
+			# Also make sure that what we have is a String, just in case
+      # a type of data different than a String has been
+			# assigned to this configuration parameter
+      config.master.metrics.transactions = config.master.metrics.transactions.to_s
+      if config.master.metrics.transactions.empty?
+        config.master.metrics.transactions = CONFIG_MASTER_METRICS_TRANSACTIONS_DEFAULT
+      end
+      config.master.metrics.transactions_authorize = config.master.metrics.transactions_authorize.to_s
+      if config.master.metrics.transactions_authorize.empty?
+        config.master.metrics.transactions_authorize = CONFIG_MASTER_METRICS_TRANSACTIONS_AUTHORIZE_DEFAULT
+      end
+
+      # can_create_event_buckets is just for our SaaS analytics system.
+      # If SaaS has been set to false, we need to disable buckets too.
+      config.can_create_event_buckets = false unless config.saas
+
+      config.stats.delete_batch_size = parse_int(config.stats.delete_batch_size,
+                                                 CONFIG_DELETE_STATS_BATCH_SIZE)
+
+      config.stats.delete_partition_batch_size = parse_int(config.stats.delete_partition_batch_size,
+                                                           CONFIG_DELETE_STATS_PARTITION_BATCH_SIZE)
+
+      # often we don't have a log_file setting - generate it here from
+      # the log_path setting.
+      log_file = config.log_file
+      if !log_file || log_file.empty?
+        log_path = config.log_path
+        config.log_file = if log_path && !log_path.empty?
+                            if File.stat(log_path).ftype == 'directory'
+                              "#{log_path}/backend_logger.log"
+                            else
+                              log_path
+                            end
+                          else
+                            ENV['CONFIG_LOG_FILE'] || STDOUT
+                          end
+      end
+    end
+  end
+end

data/lib/3scale/backend/configuration/loader.rb

@@ -0,0 +1,42 @@
+require 'ostruct'
+
+module ThreeScale
+  module Backend
+    module Configuration
+      class Loader < OpenStruct
+        Error = Class.new StandardError
+        NoConfigFiles = Class.new Error
+
+        # Add configuration section with the given fields.
+        #
+        # == Example
+        #
+        #   # Define section like this
+        #   loader = Loader.new
+        #   loader.add_section(:bacons, :type, :amount)
+        #
+        #   # Configure default values like this
+        #   loader.bacons.type   = :chunky
+        #   loader.bacons.amount = 'a lot'
+        #
+        #   # Load the configuration from an array of files
+        #   loader.load!(files)
+        #
+        #   # Use like this
+        #   loader.bacons.type # :chunky
+        #
+        def add_section(name, *fields)
+          send("#{name}=", Struct.new(*fields).new)
+        end
+
+        # Load configuration from a set of files
+        def load!(files)
+          raise NoConfigFiles if !files || files.empty?
+          files.each do |path|
+            load path if File.readable?(File.expand_path(path))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/constants.rb

@@ -0,0 +1,19 @@
+module ThreeScale
+  module Backend
+    module Constants
+      module All
+        TIME_FORMAT          = '%Y-%m-%d %H:%M:%S %z'.freeze
+        PIPELINED_SLICE_SIZE = 400
+      end
+
+      def self.included(base)
+        All.constants.each do |k|
+          base.const_set(k, All.const_get(k))
+          base.private_constant k
+        end
+      end
+    end
+
+    include Constants
+  end
+end

data/lib/3scale/backend/cors.rb

@@ -0,0 +1,84 @@
+# CORS support
+#
+# Please see references:
+#
+# https://www.w3.org/TR/cors/
+# https://code.google.com/archive/p/html5security/wikis/CrossOriginRequestSecurity.wiki
+#
+module ThreeScale
+  module Backend
+    module CORS
+      def self.stringify_consts(*consts)
+        consts.each do |k|
+          val = const_get k
+          val = val.respond_to?(:join) ? val.join(', ') : val.to_s
+          k_s = "#{k}_S".to_sym
+          const_set(k_s, val.freeze)
+          private_constant k_s
+        end
+      end
+      private_class_method :stringify_consts
+
+      MAX_AGE = 86400
+      private_constant :MAX_AGE
+
+      ALLOW_ORIGIN = '*'.freeze
+      private_constant :ALLOW_ORIGIN
+
+      ALLOW_METHODS = [
+        'GET'.freeze,
+        'POST'.freeze,
+        'PATCH'.freeze,
+        'PUT'.freeze,
+        'DELETE'.freeze
+      ].freeze
+      private_constant :ALLOW_METHODS
+
+      ALLOW_HEADERS = [
+        'Authorization'.freeze,
+        'Accept-Encoding'.freeze,
+        'Content-Type'.freeze,
+        'Cache-Control'.freeze,
+        'Accept'.freeze,
+        'If-Match'.freeze,
+        'If-Modified-Since'.freeze,
+        'If-None-Match'.freeze,
+        'If-Unmodified-Since'.freeze,
+        'X-Requested-With'.freeze,
+        'X-HTTP-Method-Override'.freeze,
+        '3scale-options'.freeze,
+      ].freeze
+      private_constant :ALLOW_HEADERS
+
+      EXPOSE_HEADERS = [
+        'ETag'.freeze,
+        'Link'.freeze,
+        '3scale-rejection-reason'.freeze,
+      ].freeze
+      private_constant :EXPOSE_HEADERS
+
+      stringify_consts :MAX_AGE, :ALLOW_METHODS, :ALLOW_HEADERS, :EXPOSE_HEADERS
+
+      HEADERS = {
+        'Access-Control-Allow-Origin'.freeze => ALLOW_ORIGIN,
+        'Access-Control-Expose-Headers'.freeze => EXPOSE_HEADERS_S,
+      }.freeze
+      private_constant :HEADERS
+
+      OPTIONS_HEADERS = {
+        'Access-Control-Max-Age'.freeze => MAX_AGE_S,
+        'Access-Control-Allow-Methods'.freeze => ALLOW_METHODS_S,
+        'Access-Control-Allow-Headers'.freeze => ALLOW_HEADERS_S,
+      }.merge(HEADERS).freeze
+      private_constant :OPTIONS_HEADERS
+
+      def self.headers
+        HEADERS
+      end
+
+      def self.options_headers
+        OPTIONS_HEADERS
+      end
+    end
+  end
+end

data/lib/3scale/backend/distributed_lock.rb

@@ -0,0 +1,67 @@
+module ThreeScale
+  module Backend
+
+    # This class uses Redis to implement a distributed lock.
+    #
+    # To implement the distributed lock, we use the Redis operation 'set nx'.
+    # The locking algorithm is detailed here: http://redis.io/topics/distlock
+    # Basically, every time that we want to use the lock, we generate a random
+    # number and set a key in Redis with that random number if its current
+    # value is null. If we could set the value, it means that we could get the
+    # lock. To release it, we just need to set to delete the same key.
+    #
+    # The implementation used in this class has some limitations.
+    # This is limited to a single Redis instance (through Twemproxy), and if
+    # the master goes off the mutual exclusion basically does not exist
+    # anymore. But there is another thing that breaks the mutual exclusion:
+    # whatever we do within the lock critical section is racing against the
+    # TTL, and we cannot guarantee that the section will be finished within the
+    # limit of the TTL. It can be the case that even if we actually locked
+    # Redis, the TTL would have expired before we got the response (think about
+    # really bad network conditions or scheduling issues in the computer that
+    # is running the critical section). So the lock acts more as an "advisory"
+    # lock than a real lock: whatever we execute inside the critical section is
+    # "probably going to be with mutual exclusion, but no guarantees".
+    #
+    # Possible ways to minimize the window of this race condition:
+    #   1) Do all the work and just lock for committing.
+    #   2) Use large values as TTLs as much as possible.
+    class DistributedLock
+      MAX_RANDOM = 1_000_000_000
+      private_constant :MAX_RANDOM
+
+      def initialize(resource, ttl, storage)
+        @resource = resource
+        @ttl = ttl
+        @storage = storage
+        @random = Random.new
+      end
+
+      # Returns key to unlock if the lock is acquired. Nil otherwise.
+      def lock
+        key = lock_key
+        storage.set(lock_storage_key, key, nx: true, ex: ttl) ? key : nil
+      end
+
+      def unlock
+        storage.del(lock_storage_key)
+      end
+
+      def current_lock_key
+        storage.get(lock_storage_key)
+      end
+
+      private
+
+      attr_reader :resource, :ttl, :storage, :random
+
+      def lock_key
+        random.rand(MAX_RANDOM).to_s
+      end
+
+      def lock_storage_key
+        "#{resource.downcase}:lock".freeze
+      end
+    end
+  end
+end