apisonator 2.100.0

data/lib/3scale/backend/statsd.rb

@@ -0,0 +1,14 @@
+module ThreeScale
+  module Backend
+    class Statsd
+      include Configurable
+
+      class << self
+        def instance
+          @instance ||= ::Statsd.new(configuration.statsd.host,
+                                     configuration.statsd.port)
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/storable.rb

@@ -0,0 +1,35 @@
+module ThreeScale
+  module Backend
+    # Mix this into objects that should be storable in the storage.
+    module Storable
+      include StorageKeyHelpers
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      def initialize(attributes = {})
+        attributes.each do |key, value|
+          send("#{key}=", value)
+        end
+      end
+
+      def storage
+        self.class.storage
+      end
+
+      module ClassMethods
+        include StorageKeyHelpers
+
+        def storage
+          Storage.instance
+        end
+
+        # returns array of attribute names
+        def attribute_names
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/storage.rb

@@ -0,0 +1,40 @@
+require '3scale/backend/storage_sync'
+
+module ThreeScale
+  module Backend
+    class Storage
+      include Configurable
+
+      # Require async code conditionally to avoid potential side-effects
+      if configuration.redis.async
+        require '3scale/backend/storage_async'
+      end
+
+      # Constant used for when batching of operations
+      # is desired/needed. Batching is performed when a lot
+      # of storage operations are need to be performed and we
+      # want to minimize database blocking of other clients
+      BATCH_SIZE = 400
+
+      class << self
+        def instance(reset = false)
+          storage_client_class.instance(reset)
+        end
+
+        def new(options)
+          storage_client_class.new(options)
+        end
+
+        private
+
+        def storage_client_class
+          if configuration.redis.async
+            Backend::StorageAsync::Client
+          else
+            Backend::StorageSync
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/storage_async.rb

@@ -0,0 +1,4 @@
+require '3scale/backend/storage_async/client'
+require '3scale/backend/storage_async/pipeline'
+require '3scale/backend/storage_async/async_redis'
+require '3scale/backend/storage_async/resque_extensions'

data/lib/3scale/backend/storage_async/async_redis.rb

@@ -0,0 +1,21 @@
+# Monkey-patches the async-redis lib to provide a 'call_pipeline' method that
+# sends multiple commands at once and returns an array of the responses for
+# each of them.
+
+module Async
+  module Redis
+    class Client
+      def call_pipeline(commands)
+        @pool.acquire do |connection|
+          commands.each do |command|
+            connection.write_request(command)
+          end
+
+          connection.flush
+
+          commands.size.times.map { connection.read_response }
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/storage_async/client.rb

@@ -0,0 +1,205 @@
+require 'async/io'
+require 'async/redis/client'
+
+module ThreeScale
+  module Backend
+    module StorageAsync
+
+      # This is a wrapper for the Async-Redis client
+      # (https://github.com/socketry/async-redis).
+      # This class overrides some methods to provide the same interface that
+      # the redis-rb client provides.
+      # This is done to avoid modifying all the model classes which assume that
+      # the Storage instance behaves likes the redis-rb client.
+      class Client
+        include Configurable
+
+        DEFAULT_HOST = 'localhost'.freeze
+        private_constant :DEFAULT_HOST
+
+        DEFAULT_PORT = 22121
+        private_constant :DEFAULT_PORT
+
+        HOST_PORT_REGEX = /redis:\/\/(.*):(\d+)/
+        private_constant :HOST_PORT_REGEX
+
+        class << self
+          attr_writer :instance
+
+          def instance(reset = false)
+            if reset || @instance.nil?
+              @instance = new(
+                  Storage::Helpers.config_with(
+                      configuration.redis,
+                      options: { default_url: "#{DEFAULT_HOST}:#{DEFAULT_PORT}" }
+                  )
+              )
+            else
+              @instance
+            end
+          end
+        end
+
+        def initialize(opts)
+          host, port = opts[:url].match(HOST_PORT_REGEX).captures if opts[:url]
+          host ||= DEFAULT_HOST
+          port ||= DEFAULT_PORT
+
+          endpoint = Async::IO::Endpoint.tcp(host, port)
+          @redis_async = Async::Redis::Client.new(endpoint)
+          @building_pipeline = false
+        end
+
+        # Now we are going to define the methods to run redis commands
+        # following the interface of the redis-rb lib.
+        #
+        # These are the different cases:
+        #   1) Methods that can be called directly. For example SET:
+        #      @redis_async.call('SET', some_key)
+        #   2) Methods that need to be "boolified". These are methods for which
+        #      redis-rb returns a boolean, but redis just returns an integer.
+        #      For example, Redis returns 0 or 1 for the EXISTS command, but
+        #      redis-rb transforms that into a boolean.
+        #   3) There are a few methods that need to be treated differently and
+        #      do not fit in any of the previous categories. For example, SSCAN
+        #      which accepts a hash of options in redis-rb.
+        #
+        # All of this might be simplified a bit in the future using the
+        # "methods" in async-redis
+        # https://github.com/socketry/async-redis/tree/master/lib/async/redis/methods
+        # but there are some commands missing, so for now, that's not an option.
+
+        METHODS_TO_BE_CALLED_DIRECTLY = [
+          :del,
+          :expire,
+          :expireat,
+          :flushdb,
+          :get,
+          :hset,
+          :hmget,
+          :incr,
+          :incrby,
+          :keys,
+          :llen,
+          :lpop,
+          :lpush,
+          :lrange,
+          :ltrim,
+          :mget,
+          :ping,
+          :rpush,
+          :scard,
+          :setex,
+          :smembers,
+          :sunion,
+          :ttl,
+          :zcard,
+          :zrangebyscore,
+          :zremrangebyscore,
+          :zrevrange
+        ].freeze
+        private_constant :METHODS_TO_BE_CALLED_DIRECTLY
+
+        METHODS_TO_BE_CALLED_DIRECTLY.each do |method|
+          define_method(method) do |*args|
+            @redis_async.call(method, *args.flatten)
+          end
+        end
+
+        METHODS_TO_BOOLIFY = [
+          :exists,
+          :sismember,
+          :sadd,
+          :srem,
+          :zadd
+        ].freeze
+        private_constant :METHODS_TO_BOOLIFY
+
+        METHODS_TO_BOOLIFY.each do |method|
+          define_method(method) do |*args|
+            @redis_async.call(method, *args.flatten) > 0
+          end
+        end
+
+        def blpop(*args)
+          call_args = ['BLPOP'] + args
+
+          # redis-rb accepts a Hash as last arg that can contain :timeout.
+          if call_args.last.is_a? Hash
+            timeout = call_args.pop[:timeout]
+            call_args << timeout
+          end
+
+          @redis_async.call(*call_args.flatten)
+        end
+
+        def set(key, val, opts = {})
+          args = ['SET', key, val]
+
+          args += ['EX', opts[:ex]] if opts[:ex]
+          args << 'NX' if opts[:nx]
+
+          @redis_async.call(*args)
+        end
+
+        def sscan(key, cursor, opts = {})
+          args = ['SSCAN', key, cursor]
+
+          args += ['MATCH', opts[:match]] if opts[:match]
+          args += ['COUNT', opts[:count]] if opts[:count]
+
+          @redis_async.call(*args)
+        end
+
+        def scan(cursor, opts = {})
+          args = ['SCAN', cursor]
+
+          args += ['MATCH', opts[:match]] if opts[:match]
+          args += ['COUNT', opts[:count]] if opts[:count]
+
+          @redis_async.call(*args)
+        end
+
+        # This method allows us to send pipelines like this:
+        # storage.pipelined do
+        #   storage.get('a')
+        #   storage.get('b')
+        # end
+        def pipelined(&block)
+          # This replaces the client with a Pipeline that accumulates the Redis
+          # commands run in a block and sends all of them in a single request.
+          #
+          # There's an important limitation: this assumes that the fiber will
+          # not yield in the block.
+
+          # When running a nested pipeline, we just need to continue
+          # accumulating commands.
+          if @building_pipeline
+            block.call
+            return
+          end
+
+          @building_pipeline = true
+
+          original = @redis_async
+          pipeline = Pipeline.new
+          @redis_async = pipeline
+
+          begin
+            block.call
+          ensure
+            @redis_async = original
+            @building_pipeline = false
+          end
+
+          pipeline.run(original)
+        end
+
+        def close
+          @redis_async.close
+        end
+      end
+
+    end
+  end
+end

data/lib/3scale/backend/storage_async/pipeline.rb

@@ -0,0 +1,79 @@
+module ThreeScale
+  module Backend
+    module StorageAsync
+
+      # This class accumulates commands and sends several of them in a single
+      # request, instead of sending them one by one.
+      class Pipeline
+
+        Error = Class.new StandardError
+
+        class PipelineSharedBetweenFibers < Error
+          def initialize
+            super 'several fibers are modifying the same Pipeline'
+          end
+        end
+
+        # There are 2 groups of commands that need to be treated a bit
+        # differently to follow the same interface as the redis-rb lib.
+        # 1) The ones that need to return a bool when redis returns "1" or "0".
+        # 2) The ones that need to return whether the result is greater than 0.
+
+        CHECK_EQUALS_ONE = %w(EXISTS SISMEMBER).freeze
+        private_constant :CHECK_EQUALS_ONE
+
+        CHECK_GREATER_THAN_0 = %w(SADD SREM ZADD).freeze
+        private_constant :CHECK_GREATER_THAN_0
+
+        def initialize
+          # Each command is an array where the first element is the name of the
+          # command ('SET', 'GET', etc.) and the rest of elements are the
+          # parameters for that command.
+          # Ex: ['SET', 'some_key', 42].
+          @commands = []
+
+          # Save the ID of the fiber that created the Pipeline so later we
+          # can check that this pipeline is not shared between fibers.
+          @fiber_id = Fiber.current.object_id
+        end
+
+        # In the async-redis lib, all the commands are run with .call:
+        # client.call('GET', 'a'), client.call('SET', 'b', '1'), etc.
+        # This method just accumulates the commands and their params.
+        def call(*args)
+          if @fiber_id != Fiber.current.object_id
+            raise PipelineSharedBetweenFibers
+          end
+
+          @commands << args
+
+          # Some Redis commands in StorageAsync compare the result with 0.
+          # For example, EXISTS. We return an integer so the comparison does
+          # not raise an error. It does not matter which int, because here we
+          # only care about adding the command to @commands.
+
+          1
+        end
+
+        # Send to redis all the accumulated commands.
+        # Returns an array with the result for each command in the same order
+        # that they added with .call().
+        def run(redis_async_client)
+          responses = redis_async_client.call_pipeline(@commands)
+
+          responses.zip(@commands).map do |resp, cmd|
+            command_name = cmd.first.to_s.upcase
+
+            if CHECK_EQUALS_ONE.include?(command_name)
+              resp.to_i == 1
+            elsif CHECK_GREATER_THAN_0.include?(command_name)
+              resp.to_i > 0
+            else
+              resp
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale/backend/storage_async/resque_extensions.rb

@@ -0,0 +1,30 @@
+# The async lib does not work well with the Resque gem. It crashes when running
+# a pipeline in the enqueue method.
+# This module mokey-patches that method.
+
+module ThreeScale
+  module Backend
+    module StorageAsync
+      module ResqueExtensions
+        def enqueue(klass, *args)
+          queue = queue_from_class(klass)
+
+          # The redis client is hidden inside a data store that contains a
+          # namespace that contains the redis client. Both vars are called
+          # "redis".
+          async_client = Resque.redis.instance_variable_get(:@redis).instance_variable_get(:@redis)
+
+          # We need to add the "resque" namespace in the keys for all the
+          # commands.
+          async_client.pipelined do
+            async_client.sadd('resque:queues', queue.to_s)
+            async_client.rpush(
+              "resque:queue:#{queue}", Resque.encode(:class => klass.to_s, :args => args)
+            )
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/3scale/backend/storage_helpers.rb

@@ -0,0 +1,278 @@
+module ThreeScale
+  module Backend
+    module StorageHelpers
+      private
+      def encode(stuff)
+        Yajl::Encoder.encode(stuff)
+      end
+
+      def decode(encoded_stuff)
+        stuff = Yajl::Parser.parse(encoded_stuff).symbolize_names
+        stuff[:timestamp] = Time.parse_to_utc(stuff[:timestamp]) if stuff[:timestamp]
+        stuff
+      end
+
+      def storage
+        Storage.instance
+      end
+    end
+
+    class Storage
+      Error = Class.new StandardError
+
+      # this is a private error
+      UnspecifiedURIScheme = Class.new Error
+      private_constant :UnspecifiedURIScheme
+
+      class UnspecifiedURI < Error
+        def initialize
+          super "Redis URL not specified with url, " \
+                  "proxy, server or default_url."
+        end
+      end
+
+      class InvalidURI < Error
+        def initialize(url, error)
+          super "The provided URL #{url.inspect} is not valid: #{error}"
+        end
+      end
+
+      module Helpers
+        class << self
+          # CONN_OPTIONS - Redis client default connection options
+          CONN_OPTIONS = {
+              connect_timeout: 5,
+              read_timeout: 3,
+              write_timeout: 3,
+              # this is set to zero to avoid potential double transactions
+              # see https://github.com/redis/redis-rb/issues/668
+              reconnect_attempts: 0,
+              # use by default the C extension client
+              driver: :hiredis
+          }.freeze
+          private_constant :CONN_OPTIONS
+
+          # CONN_WHITELIST - Connection options that can be specified in config
+          # Note: we don't expose reconnect_attempts until the bug above is fixed
+          CONN_WHITELIST = [:connect_timeout, :read_timeout, :write_timeout].freeze
+          private_constant :CONN_WHITELIST
+
+          # Parameters regarding target server we will take from a config object
+          URL_WHITELIST = [:url, :proxy, :server, :sentinels, :role].freeze
+          private_constant :URL_WHITELIST
+
+          # these are the parameters we will take from a config object
+          CONFIG_WHITELIST = (URL_WHITELIST + CONN_WHITELIST).freeze
+          private_constant :CONFIG_WHITELIST
+
+          DEFAULT_SENTINEL_PORT = 26379
+          private_constant :DEFAULT_SENTINEL_PORT
+
+          # Generate an options hash suitable for Redis.new's constructor
+          #
+          # The options hash will overwrite any settings in the configuration,
+          # and will also accept a special "default_url" parameter to apply when
+          # no URL-related parameters are passed in (both in configuration and
+          # in options).
+          #
+          # The whitelist and defaults keyword parameters control the
+          # whitelisted configuration keys to add to the Redis parameters and
+          # the default values for unspecified keys. This way you don't need to
+          # rely on hardcoded defaults.
+          def config_with(config,
+                          options: {},
+                          whitelist: CONFIG_WHITELIST,
+                          defaults: CONN_OPTIONS)
+            cfg_options = parse_dbcfg(config)
+            cfg = whitelist.each_with_object({}) do |k, h|
+              val = cfg_options[k]
+              h[k] = val if val
+            end.merge(options)
+
+            cfg_with_sentinels = cfg_sentinels_handler cfg
+
+            defaults.merge(ensure_url_param(cfg_with_sentinels))
+          end
+
+          private
+
+          # Takes an object that can be converted to a Hash and returns a
+          # suitable options hash for using with our constructor.
+          #
+          # This is intended to be called on configuration objects for our
+          # database only, since it assumes that nil values need to be thrown
+          # out, but can also be used with hashes if you are ok with removing
+          # keys with nil values.
+          #
+          # Does not modify the original object.
+          def parse_dbcfg(dbcfg)
+            if !dbcfg.is_a? Hash
+              raise "can't convert #{dbcfg.inspect} to a Hash" if !dbcfg.respond_to? :to_h
+
+              dbcfg = dbcfg.to_h
+            end
+
+            # unfortunately the current config object translates blank (not
+            # filled in) configuration options to nil, so we can't distinguish
+            # between non-specification and an actual nil value - so remove
+            # them to avoid overriding default values with nils.
+            dbcfg.reject do |_, v|
+              v.nil?
+            end
+          end
+
+          def to_redis_uri(maybe_uri)
+            raise UnspecifiedURI if maybe_uri.nil?
+            raise InvalidURI.new(maybe_uri, 'empty URL') if maybe_uri.empty?
+
+            begin
+              validate_redis_uri maybe_uri
+            rescue URI::InvalidURIError, UnspecifiedURIScheme => e
+              begin
+                validate_redis_uri 'redis://' + maybe_uri
+              rescue
+                # tag and re-raise the original error to avoid confusion
+                raise InvalidURI.new(maybe_uri, e)
+              end
+            rescue => e
+              # tag exception
+              raise InvalidURI.new(maybe_uri, e)
+            end
+          end
+
+          # Helper for the method above
+          #
+          # This raises unless maybe_uri is a valid URI.
+          def validate_redis_uri(maybe_uri)
+            # this might raise URI-specific exceptions
+            parsed_uri = URI.parse maybe_uri
+            # the parsing can succeed without scheme, so check for it and try
+            # to correct the URI.
+            raise UnspecifiedURIScheme if parsed_uri.scheme.nil?
+            # Check when host is parsed as scheme
+            raise URI::InvalidURIError if parsed_uri.host.nil? && parsed_uri.path.nil?
+
+            # return validated URI
+            maybe_uri
+          end
+
+          # This ensures we always use the :url parameter (and removes others)
+          def ensure_url_param(options)
+            proxy = options.delete :proxy
+            server = options.delete :server
+            default_url = options.delete :default_url
+
+            # order of preference: url, proxy, server, default_url
+            options[:url] = [options[:url], proxy, server, default_url].find do |val|
+              val && !val.empty?
+            end
+
+            # not having a :url parameter at this point will throw up an
+            # exception when validating the url
+            options[:url] = to_redis_uri(options[:url])
+
+            options
+          end
+
+          # Expected sentinel input cfg format:
+          #
+          # Either a String with one or more URLs:
+          #   "redis_url0,redis_url1,redis_url2,....,redis_urlN"
+          # Or an Array of Strings representing one URL each:
+          #   ["redis_url0", "redis_url1", ..., "redis_urlN"]
+          # Or an Array of Hashes with ":host", ":port", and ":password" (optional) keys:
+          #   [{ host: "srv0", port: 7379 }, { host: "srv1", port: 7379, password: "abc" }, ...]
+          #
+          # When using the String input, the comma "," character is the
+          # delimiter between URLs and the "\" character is the escaper that
+          # allows you to include commas "," and any other character verbatim in
+          # a URL.
+          #
+          # Parse to expected format by redis client
+          # [
+          #   { host: "host0", port: "port0" },
+          #   { host: "host1", port: "port1" },
+          #   { host: "host2", port: "port2", password: "abc" },
+          #   ...
+          #   { host: "hostN", port: "portN" }
+          # ]
+          def cfg_sentinels_handler(options)
+            # get role attr and remove from options
+            # will only be validated and included when sentinels are valid
+            role = options.delete :role
+            sentinels = options.delete :sentinels
+            # The Redis client can't accept empty string or array of :sentinels
+            return options if sentinels.nil? || sentinels.empty?
+
+            sentinels = Splitter.split(sentinels) if sentinels.is_a? String
+
+            options[:sentinels] = sentinels.map do |sentinel|
+              if sentinel.is_a? Hash
+                next if sentinel.empty?
+                sentinel.fetch(:host) do
+                  raise InvalidURI.new("(sentinel #{sentinel.inspect})",
+                                       'no host given')
+                end
+                sentinel
+              else
+                sentinel_to_hash sentinel
+              end
+            end.compact
+
+            # For the sentinels that do not have the :port key or
+            # the port key is nil we configure them with the default
+            # sentinel port
+            options[:sentinels].each do |sentinel|
+              sentinel[:port] ||= DEFAULT_SENTINEL_PORT
+              sentinel.delete(:password) if sentinel[:password].nil? || sentinel[:password].empty?
+            end
+
+            # Handle role option when sentinels are validated
+            options[:role] = role if role && !role.empty?
+            options
+          end
+
+          # helper to convert a sentinel object to a Hash
+          def sentinel_to_hash(sentinel)
+            return if sentinel.nil?
+
+            if sentinel.respond_to? :strip!
+              sentinel.strip!
+              # invalid string if it's empty after stripping
+              return if sentinel.empty?
+            end
+
+            valid_uri_str = to_redis_uri(sentinel)
+            # it is safe to perform URI parsing now
+            uri = URI.parse valid_uri_str
+
+            { host: uri.host, port: uri.port, password: uri.password }
+          end
+
+          # split a string by a delimiter character with escaping
+          module Splitter
+            def self.split(str, delimiter: ',', escaper: '\\')
+              escaping = false
+
+              str.each_char.inject(['']) do |ary, c|
+                if escaping
+                  escaping = false
+                  ary.last << c
+                elsif c == delimiter
+                  ary << ''
+                elsif c == escaper
+                  escaping = true
+                else
+                  ary.last << c
+                end
+
+                ary
+              end
+            end
+          end
+          private_constant :Splitter
+        end
+      end
+    end
+  end
+end