faulty 0.1.0

data/lib/faulty/storage/interface.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Faulty
+  module Storage
+    # The interface required for a storage backend implementation
+    #
+    # This is for documentation only and is not loaded
+    class Interface
+      # Add a circuit run entry to storage
+      #
+      # The backend may choose to store this in whatever manner it chooses as
+      # long as it can implement the other read methods.
+      #
+      # @param circuit [Circuit] The circuit that ran
+      # @param time [Integer] The unix timestamp for the run
+      # @param success [Boolean] True if the run succeeded
+      # @return [Status] The circuit status after the run is added
+      def entry(circuit, time, success)
+        raise NotImplementedError
+      end
+
+      # Set the circuit state to open
+      #
+      # If multiple parallel processes open the circuit simultaneously, open
+      # may be called more than once. If so, this method should return true
+      # only once, when the circuit transitions from closed to open.
+      #
+      # If the backend does not support locking or atomic operations, then
+      # it may always return true, but that could result in duplicate open
+      # notifications.
+      #
+      # If returning true, this method also updates opened_at to the
+      # current time.
+      #
+      # @param circuit [Circuit] The circuit to open
+      # @param opened_at [Integer] The timestmp the circuit was opened at
+      # @return [Boolean] True if the circuit transitioned from closed to open
+      def open(circuit, opened_at)
+        raise NotImplementedError
+      end
+
+      # Reset the opened_at time for a half_open circuit
+      #
+      # If multiple parallel processes open the circuit simultaneously, reopen
+      # may be called more than once. If so, this method should return true
+      # only once, when the circuit updates the opened_at value. It can use the
+      # value from previous_opened_at to do a compare-and-set operation.
+      #
+      # If the backend does not support locking or atomic operations, then
+      # it may always return true, but that could result in duplicate reopen
+      # notifications.
+      #
+      # @param circuit [Circuit] The circuit to reopen
+      # @param opened_at [Integer] The timestmp the circuit was opened at
+      # @param previous_opened_at [Integer] The last known value of opened_at.
+      #   Can be used to comare-and-set.
+      # @return [Boolean] True if the opened_at time was updated
+      def reopen(circuit, opened_at, previous_opened_at)
+        raise NotImplementedError
+      end
+
+      # Set the circuit state to closed
+      #
+      # If multiple parallel processes close the circuit simultaneously, close
+      # may be called more than once. If so, this method should return true
+      # only once, when the circuit transitions from open to closed.
+      #
+      # If the backend does not support locking or atomic operations, then
+      # it may always return true, but that could result in duplicate close
+      # notifications.
+      #
+      # @return [Boolean] True if the circuit transitioned from open to closed
+      def close(circuit)
+        raise NotImplementedError
+      end
+
+      # Lock the circuit in a given state
+      #
+      # No concurrency gurantees are provided for locking
+      #
+      # @param circuit [Circuit] The circuit to lock
+      # @param state [:open, :closed] The state to lock the circuit in
+      # @return [void]
+      def lock(circuit, state)
+        raise NotImplementedError
+      end
+
+      # Unlock the circuit from any state
+      #
+      # No concurrency gurantees are provided for locking
+      #
+      # @param circuit [Circuit] The circuit to unlock
+      # @return [void]
+      def unlock(circuit)
+        raise NotImplementedError
+      end
+
+      # Reset the circuit to a fresh state
+      #
+      # Clears all circuit status including entries, state, locks,
+      # opened_at, and any other values that would affect Status.
+      #
+      # No concurrency gurantees are provided for resetting
+      #
+      # @param circuit [Circuit] The circuit to unlock
+      # @return [void]
+      def reset(circuit)
+        raise NotImplementedError
+      end
+
+      # Get the status object for a circuit
+      #
+      # No concurrency gurantees are provided for getting status. It's possible
+      # that status may represent a circuit in the middle of modification.
+      #
+      # @param circuit [Circuit] The circuit to get status for
+      # @return [Status] The current status
+      def status(circuit)
+        raise NotImplementedError
+      end
+
+      # Get the entry history of a circuit
+      #
+      # No concurrency gurantees are provided for getting status. It's possible
+      # that status may represent a circuit in the middle of modification.
+      #
+      # A storage backend may choose not to implement this method and instead
+      # return an empty array.
+      #
+      # Each item in the history array is an array of two items (a tuple) of
+      # `[run_time, succeeded]`, where `run_time` is a unix timestamp, and
+      # `succeeded` is a boolean, true if the run succeeded.
+      #
+      # @param circuit [Circuit] The circuit to get history for
+      # @return [Array<Array>] An array of history tuples
+      def history(circuit)
+        raise NotImplementedError
+      end
+
+      # Get a list of all circuit names
+      #
+      # If the storage backend does not support listing circuits, this may
+      # return an empty array.
+      #
+      # @return [Array<String>]
+      def list
+        raise NotImplementedError
+      end
+
+      # Can this storage backend raise an error?
+      #
+      # If the storage backend returns false from this method, it will be wrapped
+      # in a {FaultTolerantProxy}, otherwise it will be used as-is.
+      #
+      # @return [Boolean] True if this cache backend is fault tolerant
+      def fault_tolerant?
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/faulty/storage/memory.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Faulty
+  module Storage
+    # The default in-memory storage for circuits
+    #
+    # This implementation is most suitable to single-process, low volume
+    # usage. It is thread-safe and circuit state is shared across threads.
+    #
+    # Circuit state and runs are stored in memory. Although runs have a maximum
+    # size within a circuit, there is no limit on the number of circuits that
+    # can be stored. This means the user should be careful about the number of
+    # circuits that are created. To that end, it's a good idea to avoid
+    # dynamically-named circuits with this backend.
+    #
+    # For a more robust distributed implementation, use the {Redis} storage
+    # backend.
+    #
+    # This can be used as a reference implementation for storage backends that
+    # store a list of circuit run entries.
+    class Memory
+      attr_reader :options
+
+      # Options for {Memory}
+      #
+      # @!attribute [r] max_sample_size
+      #   @return [Integer] The number of cache run entries to keep in memory
+      #     for each circuit. Default `100`.
+      Options = Struct.new(:max_sample_size) do
+        include ImmutableOptions
+
+        private
+
+        def defaults
+          { max_sample_size: 100 }
+        end
+      end
+
+      # The internal object for storing a circuit
+      #
+      # @private
+      MemoryCircuit = Struct.new(:state, :runs, :opened_at, :lock) do
+        def initialize
+          self.state = Concurrent::Atom.new(:closed)
+          self.runs = Concurrent::MVar.new([], dup_on_deref: true)
+          self.opened_at = Concurrent::Atom.new(nil)
+          self.lock = nil
+        end
+
+        # Create a status object from the current circuit state
+        #
+        # @param circuit_options [Circuit::Options] The circuit options object
+        # @return [Status] The newly created status
+        def status(circuit_options)
+          status = nil
+          runs.borrow do |locked_runs|
+            status = Faulty::Status.from_entries(
+              locked_runs,
+              state: state.value,
+              lock: lock,
+              opened_at: opened_at.value,
+              options: circuit_options
+            )
+          end
+
+          status
+        end
+      end
+
+      # @param options [Hash] Attributes for {Options}
+      # @yield [Options] For setting options in a block
+      def initialize(**options, &block)
+        @circuits = Concurrent::Map.new
+        @options = Options.new(options, &block)
+      end
+
+      # Add an entry to storage
+      #
+      # @see Interface#entry
+      # @param (see Interface#entry)
+      # @return (see Interface#entry)
+      def entry(circuit, time, success)
+        memory = fetch(circuit)
+        memory.runs.borrow do |runs|
+          runs.push([time, success])
+          runs.pop if runs.size > options.max_sample_size
+        end
+        memory.status(circuit.options)
+      end
+
+      # Mark a circuit as open
+      #
+      # @see Interface#open
+      # @param (see Interface#open)
+      # @return (see Interface#open)
+      def open(circuit, opened_at)
+        memory = fetch(circuit)
+        opened = memory.state.compare_and_set(:closed, :open)
+        memory.opened_at.reset(opened_at) if opened
+        opened
+      end
+
+      # Mark a circuit as reopened
+      #
+      # @see Interface#reopen
+      # @param (see Interface#reopen)
+      # @return (see Interface#reopen)
+      def reopen(circuit, opened_at, previous_opened_at)
+        memory = fetch(circuit)
+        memory.opened_at.compare_and_set(previous_opened_at, opened_at)
+      end
+
+      # Mark a circuit as closed
+      #
+      # @see Interface#close
+      # @param (see Interface#close)
+      # @return (see Interface#close)
+      def close(circuit)
+        memory = fetch(circuit)
+        memory.runs.modify { |_old| [] }
+        memory.state.compare_and_set(:open, :closed)
+      end
+
+      # Lock a circuit open or closed
+      #
+      # @see Interface#lock
+      # @param (see Interface#lock)
+      # @return (see Interface#lock)
+      def lock(circuit, state)
+        memory = fetch(circuit)
+        memory.lock = state
+      end
+
+      # Unlock a circuit
+      #
+      # @see Interface#unlock
+      # @param (see Interface#unlock)
+      # @return (see Interface#unlock)
+      def unlock(circuit)
+        memory = fetch(circuit)
+        memory.lock = nil
+      end
+
+      # Reset a circuit
+      #
+      # @see Interface#reset
+      # @param (see Interface#reset)
+      # @return (see Interface#reset)
+      def reset(circuit)
+        @circuits.delete(circuit.name)
+      end
+
+      # Get the status of a circuit
+      #
+      # @see Interface#status
+      # @param (see Interface#status)
+      # @return (see Interface#status)
+      def status(circuit)
+        fetch(circuit).status(circuit.options)
+      end
+
+      # Get the circuit history up to `max_sample_size`
+      #
+      # @see Interface#history
+      # @param (see Interface#history)
+      # @return (see Interface#history)
+      def history(circuit)
+        fetch(circuit).runs.value
+      end
+
+      # Get a list of circuit names
+      #
+      # @return [Array<String>] The circuit names
+      def list
+        @circuits.keys
+      end
+
+      # Memory storage is fault-tolerant by default
+      #
+      # @return [true]
+      def fault_tolerant?
+        true
+      end
+
+      private
+
+      # Fetch circuit storage safely or create it if it doesn't exist
+      #
+      # @return [MemoryCircuit]
+      def fetch(circuit)
+        @circuits.compute_if_absent(circuit.name) { MemoryCircuit.new }
+      end
+    end
+  end
+end

data/lib/faulty/storage/redis.rb

@@ -0,0 +1,335 @@
+# frozen_string_literal: true
+
+module Faulty
+  module Storage
+    class Redis # rubocop:disable Metrics/ClassLength
+      # Separates the time/status for history entry strings
+      ENTRY_SEPARATOR = ':'
+
+      attr_reader :options
+
+      # Options for {Redis}
+      #
+      # @!attribute [r] client
+      #   @return [Redis,ConnectionPool] The Redis instance or a ConnectionPool
+      #     used to connect to Redis. Default `::Redis.new`
+      # @!attribute [r] key_prefix
+      #   @return [String] A string prepended to all Redis keys used to store
+      #     circuit state. Default `faulty`.
+      # @!attribute [r] key_separator
+      #   @return [String] A string used to separate the parts of the Redis keys
+      #     used to store circuit state. Defaulty `:`.
+      # @!attribute [r] max_sample_size
+      #   @return [Integer] The number of cache run entries to keep in memory
+      #     for each circuit. Default `100`.
+      # @!attribute [r] sample_ttl
+      #   @return [Integer] The maximum number of seconds to store a
+      #     circuit run history entry. Default `100`.
+      # @!attribute [r] circuit_ttl
+      #   @return [Integer] The maximum number of seconds to keep a circuit.
+      #     A value of `nil` disables circuit expiration.
+      #     Default `604_800` (1 week).
+      # @!attribute [r] list_granularity
+      #   @return [Integer] The number of seconds after which a new set is
+      #     created to store circuit names. The old set is kept until
+      #     circuit_ttl expires. Default `3600` (1 hour).
+      Options = Struct.new(
+        :client,
+        :key_prefix,
+        :key_separator,
+        :max_sample_size,
+        :sample_ttl,
+        :circuit_ttl,
+        :list_granularity
+      ) do
+        include ImmutableOptions
+
+        private
+
+        def defaults
+          {
+            key_prefix: 'faulty',
+            key_separator: ':',
+            max_sample_size: 100,
+            sample_ttl: 1800,
+            circuit_ttl: 604_800,
+            list_granularity: 3600
+          }
+        end
+
+        def required
+          %i[list_granularity]
+        end
+
+        def finalize
+          self.client = ::Redis.new unless client
+        end
+      end
+
+      # @param options [Hash] Attributes for {Options}
+      # @yield [Options] For setting options in a block
+      def initialize(**options, &block)
+        @options = Options.new(options, &block)
+      end
+
+      # Add an entry to storage
+      #
+      # @see Interface#entry
+      # @param (see Interface#entry)
+      # @return (see Interface#entry)
+      def entry(circuit, time, success)
+        key = entries_key(circuit)
+        pipe do |r|
+          r.sadd(list_key, circuit.name)
+          r.expire(list_key, options.circuit_ttl + options.list_granularity) if options.circuit_ttl
+          r.lpush(key, "#{time}#{ENTRY_SEPARATOR}#{success ? 1 : 0}")
+          r.ltrim(key, 0, options.max_sample_size - 1)
+          r.expire(key, options.sample_ttl) if options.sample_ttl
+        end
+
+        status(circuit)
+      end
+
+      # Mark a circuit as open
+      #
+      # @see Interface#open
+      # @param (see Interface#open)
+      # @return (see Interface#open)
+      def open(circuit, opened_at)
+        redis do |r|
+          opened = compare_and_set(r, state_key(circuit), ['closed', nil], 'open')
+          r.set(opened_at_key(circuit), opened_at, ex: options.circuit_ttl) if opened
+          opened
+        end
+      end
+
+      # Mark a circuit as reopened
+      #
+      # @see Interface#reopen
+      # @param (see Interface#reopen)
+      # @return (see Interface#reopen)
+      def reopen(circuit, opened_at, previous_opened_at)
+        redis do |r|
+          compare_and_set(r, opened_at_key(circuit), [previous_opened_at.to_s], opened_at)
+        end
+      end
+
+      # Mark a circuit as closed
+      #
+      # @see Interface#close
+      # @param (see Interface#close)
+      # @return (see Interface#close)
+      def close(circuit)
+        redis do |r|
+          closed = compare_and_set(r, state_key(circuit), ['open'], 'closed')
+          r.del(entries_key(circuit)) if closed
+          closed
+        end
+      end
+
+      # Lock a circuit open or closed
+      #
+      # The circuit_ttl does not apply to locks
+      #
+      # @see Interface#lock
+      # @param (see Interface#lock)
+      # @return (see Interface#lock)
+      def lock(circuit, state)
+        redis { |r| r.set(lock_key(circuit), state) }
+      end
+
+      # Unlock a circuit
+      #
+      # @see Interface#unlock
+      # @param (see Interface#unlock)
+      # @return (see Interface#unlock)
+      def unlock(circuit)
+        redis { |r| r.del(lock_key(circuit)) }
+      end
+
+      # Reset a circuit
+      #
+      # @see Interface#reset
+      # @param (see Interface#reset)
+      # @return (see Interface#reset)
+      def reset(circuit)
+        pipe do |r|
+          r.del(
+            entries_key(circuit),
+            opened_at_key(circuit),
+            lock_key(circuit)
+          )
+          r.set(state_key(circuit), 'closed', ex: options.circuit_ttl)
+        end
+      end
+
+      # Get the status of a circuit
+      #
+      # @see Interface#status
+      # @param (see Interface#status)
+      # @return (see Interface#status)
+      def status(circuit)
+        futures = {}
+        pipe do |r|
+          futures[:state] = r.get(state_key(circuit))
+          futures[:lock] = r.get(lock_key(circuit))
+          futures[:opened_at] = r.get(opened_at_key(circuit))
+          futures[:entries] = r.lrange(entries_key(circuit), 0, -1)
+        end
+
+        Faulty::Status.from_entries(
+          map_entries(futures[:entries].value),
+          state: futures[:state].value&.to_sym || :closed,
+          lock: futures[:lock].value&.to_sym,
+          opened_at: futures[:opened_at].value ? futures[:opened_at].value.to_i : nil,
+          options: circuit.options
+        )
+      end
+
+      # Get the circuit history up to `max_sample_size`
+      #
+      # @see Interface#history
+      # @param (see Interface#history)
+      # @return (see Interface#history)
+      def history(circuit)
+        entries = redis { |r| r.lrange(entries_key(circuit), 0, -1) }
+        map_entries(entries).reverse
+      end
+
+      def list
+        redis { |r| r.sunion(*all_list_keys) }
+      end
+
+      # Redis storage is not fault-tolerant
+      #
+      # @return [true]
+      def fault_tolerant?
+        false
+      end
+
+      private
+
+      # Generate a key from its parts
+      #
+      # @return [String] The key
+      def key(*parts)
+        [options.key_prefix, *parts].join(options.key_separator)
+      end
+
+      def ckey(circuit, *parts)
+        key('circuit', circuit.name, *parts)
+      end
+
+      # @return [String] The key for circuit state
+      def state_key(circuit)
+        ckey(circuit, 'state')
+      end
+
+      # @return [String] The key for circuit run history entries
+      def entries_key(circuit)
+        ckey(circuit, 'entries')
+      end
+
+      # @return [String] The key for circuit locks
+      def lock_key(circuit)
+        ckey(circuit, 'lock')
+      end
+
+      # @return [String] The key for circuit opened_at
+      def opened_at_key(circuit)
+        ckey(circuit, 'opened_at')
+      end
+
+      # Get the current key to add circuit names to
+      def list_key
+        key('list', current_list_block)
+      end
+
+      # Get all active circuit list keys
+      #
+      # We use a rolling list of redis sets to store circuit names. This way we
+      # can maintain this index, while still using Redis to expire old circuits.
+      # Whenever we add a circuit to the list, we add it to the current set. A
+      # new set is created every `options.list_granularity` seconds.
+      #
+      # When reading the list, we union all sets together, which gets us the
+      # full list.
+      #
+      # Each set has its own expiration, so that the oldest sets will
+      # automatically be deleted from Redis after `options.circuit_ttl`.
+      #
+      # It is possible for a single circuit name to be a part of many of these
+      # sets. This is the space trade-off we make in exchange for automatic
+      # expiration.
+      #
+      # @return [Array<String>] An array of redis keys for circuit name sets
+      def all_list_keys
+        num_blocks = (options.circuit_ttl.to_f / options.list_granularity).floor + 1
+        start_block = current_list_block - num_blocks + 1
+        num_blocks.times.map do |i|
+          key('list', start_block + i)
+        end
+      end
+
+      # Get the block number for the current list set
+      #
+      # @return [Integer] The current block number
+      def current_list_block
+        (Faulty.current_time.to_f / options.list_granularity).floor
+      end
+
+      # Set a value in Redis only if it matches a list of current values
+      #
+      # @param redis [Redis] The redis connection
+      # @param key [String] The redis key to CAS
+      # @param old [Array<String>] A list of previous values that pass the
+      #   comparison
+      # @param new [String] The new value to set if the compare passes
+      # @return [Boolean] True if the value was set to `new`, false if the CAS
+      #   failed
+      def compare_and_set(redis, key, old, new)
+        result = redis.watch(key) do
+          if old.include?(redis.get(key))
+            redis.multi { |m| m.set(key, new) }
+          else
+            redis.unwatch
+          end
+        end
+
+        result[0] == 'OK'
+      end
+
+      # Yield a Redis connection
+      #
+      # @yield [Redis] Yields the connection to the block
+      # @return The value returned from the block
+      def redis
+        if options.client.respond_to?(:with)
+          options.client.with { |redis| yield redis }
+        else
+          yield options.client
+        end
+      end
+
+      # Yield a pipelined Redis connection
+      #
+      # @yield [Redis::Pipeline] Yields the connection to the block
+      # @return [void]
+      def pipe
+        redis { |r| r.pipelined { |p| yield p } }
+      end
+
+      # Map raw Redis history entries to Faulty format
+      #
+      # @see Storage::Interface
+      # @param raw_entries [Array<String>] The raw Redis entries
+      # @return [Array<Array>] The Faulty-formatted entries
+      def map_entries(raw_entries)
+        raw_entries.map do |e|
+          time, state = e.split(ENTRY_SEPARATOR)
+          [time.to_i, state == '1']
+        end
+      end
+    end
+  end
+end