hoodoo 1.13.0 → 1.14.0

data/lib/hoodoo/transient_store/transient_store.rb

@@ -0,0 +1,344 @@
+########################################################################
+# File::    transient_store.rb
+# (C)::     Loyalty New Zealand 2017
+#
+# Purpose:: Provide a simple abstraction over transient storage engines
+#           such as Memcached or Redis, making it easier for client code
+#           to switch engines with very few changes.
+# ----------------------------------------------------------------------
+#           01-Feb-2017 (ADH): Created.
+########################################################################
+
+module Hoodoo
+
+  # A simple abstraction over transient storage engines such as
+  # {Memcached}[https://memcached.org] or {Redis}[https://redis.io], making it
+  # it easier for client code to switch engines with very few changes. If the
+  # storage engine chosen when creating instances of this object is defined in
+  # application-wide configuration data, all you would need to do is change the
+  # configuration for all new TransientStore instances to use the new engine.
+  #
+  class TransientStore
+
+    ###########################################################################
+    # Class methods
+    ###########################################################################
+
+    # Register a new storage engine plugin class. It _MUST_ inherit from and
+    # thus follow the template laid out in Hoodoo::TransientStore::Base.
+    #
+    # _Named_ parameters are:
+    #
+    # +as+::    The name, as a Symbol, for the ::new +storage_engine+
+    #           input parameter, to select this plugin.
+    #
+    # +using+:: The class reference of the Hoodoo::TransientStore::Base
+    #           subclass to be associated with the name given in +as+.
+    #
+    # Example:
+    #
+    #     Hoodoo::TransientStore.register(
+    #       as:    :memcached,
+    #       using: Hoodoo::TransientStore::Memcached
+    #     )
+    #
+    def self.register( as:, using: )
+      as = as.to_sym
+
+      @@supported_storage_engines = {} unless defined?( @@supported_storage_engines )
+
+      unless using < Hoodoo::TransientStore::Base
+        raise "Hoodoo::TransientStore.register requires a Hoodoo::TransientStore::Base subclass - got '#{ using.to_s }'"
+      end
+
+      if @@supported_storage_engines.has_key?( as )
+        raise "Hoodoo::TransientStore.register: A storage engine called '#{ as }' is already registered"
+      end
+
+      @@supported_storage_engines[ as ] = using
+    end
+
+    # Remove a storage engine plugin class from the supported collection. Any
+    # existing Hoodoo::TransientStore instances using the removed class will
+    # not be affected, but new instances cannot be made.
+    #
+    # _Named_ parameters are:
+    #
+    # +as+:: The value given to #register in the corresponding +as+ parameter.
+    #
+    def self.deregister( as: )
+      @@supported_storage_engines.delete( as )
+    end
+
+    # Return an array of the names of all supported storage engine names known
+    # to the Hoodoo::TransientStore class. Any one of those names can be used
+    # with the ::new +storage_engine+ parameter.
+    #
+    def self.supported_storage_engines
+      @@supported_storage_engines.keys()
+    end
+
+    ###########################################################################
+    # Instance methods
+    ###########################################################################
+
+    # Read this instance's storage engine; see ::supported_storage_engines and
+    # ::new.
+    #
+    attr_reader :storage_engine
+
+    # Read the storage engine insteance for the #storage_engine - this allows
+    # engine-specific configuration to be set where available, though this is
+    # strongly discouraged as it couples client code to the engine in use,
+    # defeating the main rationale behind the TransientStore abstraction.
+    #
+    attr_reader :storage_engine_instance
+
+    # Read this instance's default item maximum lifespan, in seconds. See
+    # also ::new.
+    #
+    attr_reader :default_maximum_lifespan
+
+    # Read this instance's default storage namespace, as a String. See also
+    # ::new.
+    #
+    attr_reader :default_namespace
+
+    # Instantiate a new Transient storage object through which temporary data
+    # can be stored or retrieved.
+    #
+    # The TransientStore abstraction is a high level and simple abstraction over
+    # heterogenous data storage engines. It does not expose the many subtle
+    # configuration settings usually available in such. If you need to take
+    # advantage of those at an item storage level, you'll need to use a lower
+    # level interface and thus lock your code to the engine of choice.
+    #
+    # Engine plug-ins are recommended to attempt to gain and test a connection
+    # to the storage engine when this object is constructed, so if building a
+    # TransientStore instance, ensure your chosen storage engine is running
+    # first. Exceptions may be raised by storage engines, so you will probably
+    # want to catch those with more civilised error handling code.
+    #
+    # _Named_ parameters are:
+    #
+    # +storage_engine+::           An entry from ::supported_storage_engines.
+    #
+    # +storage_host_uri+::         The engine-dependent connection URI. Consult
+    #                              documentation for your chosen engine to find
+    #                              out its connection URI requirements, along
+    #                              with the documentation for the constructor
+    #                              method of the plug-in in use, since in some
+    #                              cases requirements may be unusual (e.g. in
+    #                              Hoodoo::TransientStore::MemcachedRedisMirror).
+    #
+    # +default_maximum_lifespan+:: The default time-to-live for data items, in,
+    #                              seconds; can be overridden per item; default
+    #                              is 604800 seconds or 7 days.
+    #
+    # +default_namespace+::        Storage engine keys are namespaced with
+    #                              +nz_co_loyalty_hoodoo_transient_store_+ by
+    #                              default, though this can be overridden here.
+    #                              Pass a String or Symbol.
+    #
+    def initialize(
+      storage_engine:,
+      storage_host_uri:,
+      default_maximum_lifespan: 604800,
+      default_namespace:        'nz_co_loyalty_hoodoo_transient_store_'
+    )
+
+      unless self.class.supported_storage_engines().include?( storage_engine )
+
+        # Be kind and use 'inspect' to indicate that we expect Symbols here
+        # in the exception, because of the arising leading ':' in the output.
+        #
+        engines = self.class.supported_storage_engines().map { | symbol | "'#{ symbol.inspect }'" }
+        allowed = engines.join( ', ' )
+
+        raise "Hoodoo::TransientStore: Unrecognised storage engine '#{ storage_engine.inspect }' requested; allowed values: #{ allowed }"
+      end
+
+      @default_maximum_lifespan = default_maximum_lifespan
+      @default_namespace        = ( default_namespace || '' ).to_s()
+      @storage_engine           = storage_engine
+      @storage_engine_instance  = @@supported_storage_engines[ storage_engine ].new(
+        storage_host_uri: storage_host_uri,
+        namespace:        @default_namespace
+      )
+
+    end
+
+    # Set (write) a given payload into the storage engine with the given
+    # payload and maximum lifespan.
+    #
+    # Payloads must only contain simple types such as Hash, Array, String and
+    # Integer. Complex types like Symbol, Date, Float, BigDecimal or custom
+    # objects are unlikely to serialise properly but since this depends upon
+    # the storage engine in use, errors may or may not be raised for misuse.
+    #
+    # Storage engines usually have a maximum payload size limit; consult your
+    # engine administrator for information. For example, the default - but
+    # reconfigurable - maximum payload size for Memcached is 1MB.
+    #
+    # For maximum possible compatibility:
+    #
+    # * Use only Hash payloads with String key/value paids and no nesting. You
+    #   may choose to marshal the data into a String manually for unusual data
+    #   requirements, manually converting back when reading stored data.
+    #
+    # * Keep the payload size as small as possible - large objects belong in
+    #   bulk storage engines such as Amazon S3.
+    #
+    # These are only guidelines though - heterogenous storage engine support
+    # and the ability of system administrators to arbitrarily configure those
+    # storage engines makes it impossible to be more precise.
+    #
+    # Returns:
+    #
+    # * +true+ if storage was successful
+    # * +false+ if storage failed but the reason is unknown
+    # * An +Exception+ instance if storage failed and the storage engine
+    #   raised an exception describing the problem.
+    #
+    # _Named_ parameters are:
+    #
+    # +key+::              Storage key to use in the engine, which is then used
+    #                      in subsequent calls to #get and possibly eventually
+    #                      to #delete. Only non-empty Strings or Symbols are
+    #                      permitted, else an exception will be raised.
+    #
+    # +payload+::          Payload data to store under the given +key+. A flat
+    #                      Hash is recommended rather than simple types such as
+    #                      String (unless marshalling a complex type into such)
+    #                      in order to make potential additions to stored data
+    #                      easier to implement. Note that +nil+ is prohibited.
+    #
+    # +maximum_lifespan+:: Optional maximum lifespan, seconds. Storage engines
+    #                      may chooset to evict payloads sooner than this; it
+    #                      is a maximum time, not a guarantee. Omit to use this
+    #                      TransientStore instance's default value - see ::new.
+    #                      If you know you no longer need a piece of data at a
+    #                      particular point in the execution flow of your code,
+    #                      explicitly delete it via #delete rather than leaving
+    #                      it to expire. This maximises the storage engine's
+    #                      pool free space and so minimises the chance of early
+    #                      item eviction.
+    #
+    def set( key:, payload:, maximum_lifespan: nil )
+      key = normalise_key( key, 'set' )
+
+      if payload.nil?
+        raise "Hoodoo::TransientStore\#set: Payloads of 'nil' are prohibited"
+      end
+
+      maximum_lifespan ||= @default_maximum_lifespan
+
+      begin
+        result = @storage_engine_instance.set(
+          key:              key,
+          payload:          payload,
+          maximum_lifespan: maximum_lifespan
+        )
+
+        if result != true && result != false
+          raise "Hoodoo::TransientStore\#set: Engine '#{ @storage_engine }' returned an invalid response"
+        end
+
+      rescue => e
+        result = e
+
+      end
+
+      return result
+    end
+
+    # Retrieve data previously stored with #set.
+    #
+    # _Named_ parameters are:
+    #
+    # +key+:: Key previously given to #set.
+    #
+    # Returns +nil+ if the item is not found - either the key is wrong, the
+    # stored data has expired or the stored data has been evicted early from
+    # the storage engine's pool.
+    #
+    # Only non-empty String or Symbol keys are permitted, else an exception
+    # will be raised.
+    #
+    def get( key: )
+      key = normalise_key( key, 'get' )
+      @storage_engine_instance.get( key: key ) rescue nil
+    end
+
+    # Delete data previously stored with #set.
+    #
+    # _Named_ parameters are:
+    #
+    # +key+:: Key previously given to #set.
+    #
+    # Returns:
+    #
+    # * +true+ if deletion was successful, if the item has already expired or
+    #   if the key is simply not recognised so there is no more work to do.
+    # * +false+ if deletion failed but the reason is unknown.
+    # * An +Exception+ instance if deletion failed and the storage engine
+    #   raised an exception describing the problem.
+    #
+    # Only non-empty String or Symbol keys are permitted, else an exception
+    # will be raised.
+    #
+    def delete( key: )
+      key = normalise_key( key, 'delete' )
+
+      begin
+        result = @storage_engine_instance.delete( key: key )
+
+        if result != true && result != false
+          raise "Hoodoo::TransientStore\#delete: Engine '#{ @storage_engine }' returned an invalid response"
+        end
+
+      rescue => e
+        result = e
+
+      end
+
+      return result
+    end
+
+    # If you aren't going to use this instance again, it is good manners to
+    # immediately close its connection(s) to any storage engines by calling
+    # here.
+    #
+    # No useful return value is generated and exceptions are ignored.
+    #
+    def close
+      @storage_engine_instance.close() rescue nil
+    end
+
+  private
+
+    # Given a storage key, make sure it's a String or Symbol, coerce to a
+    # String and ensure it isn't empty. Returns the non-empty String version.
+    # Raises exceptions for bad input classes or empty keys.
+    #
+    # +key+::                 Key to normalise.
+    #
+    # +calling_method_name+:: Name of calling method to declare in exception
+    #                         messages, to aid callers in debugging.
+    #
+    def normalise_key( key, calling_method_name )
+      unless key.is_a?( String ) || key.is_a?( Symbol )
+        raise "Hoodoo::TransientStore\##{ calling_method_name }: Keys must be of String or Symbol class; you provided '#{ key.class }'"
+      end
+
+      key = key.to_s
+
+      if key.empty?
+        raise "Hoodoo::TransientStore\##{ calling_method_name }: Empty String or Symbol keys are prohibited"
+      end
+
+      return key
+    end
+
+  end
+end

data/lib/hoodoo/transient_store/transient_store/base.rb

@@ -0,0 +1,81 @@
+########################################################################
+# File::    base.rb
+# (C)::     Loyalty New Zealand 2017
+#
+# Purpose:: Base class for Base class for Hoodoo::TransientStore plugins.
+# ----------------------------------------------------------------------
+#           01-Feb-2017 (ADH): Created.
+########################################################################
+
+module Hoodoo
+  class TransientStore
+
+    # Base class for Hoodoo::TransientStore plugins. This is in effect just a
+    # template / abstract class, providing a source-level guideline for plug-in
+    # authors. See also out-of-the-box existing plug-ins as worked examples.
+    #
+    class Base
+
+      # Base class template for a constructor. Subclasses should try to
+      # establish a connection with their storage engine(s) here and raise
+      # exceptions if things go wrong.
+      #
+      # +storage_host_uri+:: The engine-dependent connection URI. See
+      #                      Hoodoo::TransientStore::new for details.
+      #
+      # +namespace+::        The storage key namespace to use, as a String.
+      #                      See Hoodoo::TransientStore::new for details.
+      #
+      def initialize( storage_host_uri:, namespace: )
+        @storage_host_uri = storage_host_uri
+        @namespace        = namespace
+      end
+
+      # Base class template for the plug-in's back-end implementation of
+      # Hoodoo::TransientStore#set - see that for details.
+      #
+      # The implementation is free to raise an exception if an error is
+      # encountered while trying to set the data - this will be caught and
+      # returned by Hoodoo::TransientStore#set. Otherwise return +true+ on
+      # success or +false+ for failures of unknown origin.
+      #
+      def set( key:, payload:, maximum_lifespan: )
+        raise 'Subclasses must implement Hoodoo::TransientStore::Base#set'
+      end
+
+      # Base class template for the plug-in's back-end implementation of
+      # Hoodoo::TransientStore#get - see that for details. Returns +nil+ if
+      # no data is found for the given key, or if data is explicitly +nil+.
+      #
+      # The implementation is free to raise an exception if an error is
+      # encountered while trying to get the data - this will be caught and
+      # +nil+ returned by Hoodoo::TransientStore#get.
+      #
+      def get( key: )
+        raise 'Subclasses must implement Hoodoo::TransientStore::Base#get'
+      end
+
+      # Base class template for the plug-in's back-end implementation of
+      # Hoodoo::TransientStore#delete - see that for details.
+      #
+      # The implementation is free to raise an exception if an error is
+      # encountered while trying to get the data - this will be caught and
+      # ignored by Hoodoo::TransientStore#delete. Otherwise return +true+ on
+      # success or +false+ for failures of unknown origin.
+      #
+      def delete( key: )
+        raise 'Subclasses must implement Hoodoo::TransientStore::Base#delete'
+      end
+
+      # Base class template for the plug-in's back-end implementation of
+      # Hoodoo::TransientStore#close - see that for details.
+      #
+      # Any exception raised will be ignored by Hoodoo::TransientStore#close.
+      #
+      def close
+        raise 'Subclasses must implement Hoodoo::TransientStore::Base#close'
+      end
+
+    end
+  end
+end

data/lib/hoodoo/transient_store/transient_store/memcached.rb

@@ -0,0 +1,116 @@
+########################################################################
+# File::    memcached.rb
+# (C)::     Loyalty New Zealand 2017
+#
+# Purpose:: Hoodoo::TransientStore plugin supporting Memcached.
+# ----------------------------------------------------------------------
+#           01-Feb-2017 (ADH): Created.
+########################################################################
+
+begin
+  require 'dalli'
+rescue LoadError
+end
+
+module Hoodoo
+  class TransientStore
+
+    # Hoodoo::TransientStore plugin for {Memcached}[https://memcached.org]. The
+    # {Dalli gem}[https://github.com/petergoldstein/dalli] is used for server
+    # communication.
+    #
+    class Memcached < Hoodoo::TransientStore::Base
+
+      # See Hoodoo::TransientStore::Base::new for details.
+      #
+      # Do not instantiate this class directly. Use
+      # Hoodoo::TransientStore::new.
+      #
+      # The {Dalli gem}[https://github.com/petergoldstein/dalli] is used to
+      # talk to {Memcached}[https://memcached.org] and accepts connection UIRs
+      # of simple, terse forms such as <tt>'localhost:11211'</tt>. Connections
+      # are configured with JSON serialisation, compression off and a forced
+      # namespace of +nz_co_loyalty_hoodoo_transient_store_+ to avoid collision
+      # of data stored with this object and other data that may be in the
+      # Memcached instance identified by +storage_host_uri+.
+      #
+      def initialize( storage_host_uri:, namespace: )
+        super # Pass all arguments through -> *not* 'super()'
+        @client = connect_to_memcached( storage_host_uri, namespace )
+      end
+
+      # See Hoodoo::TransientStore::Base#set for details.
+      #
+      def set( key:, payload:, maximum_lifespan: )
+        @client.set( key, payload, maximum_lifespan )
+        true
+      end
+
+      # See Hoodoo::TransientStore::Base#get for details.
+      #
+      def get( key: )
+        @client.get( key )
+      end
+
+      # See Hoodoo::TransientStore::Base#delete for details.
+      #
+      def delete( key: )
+        @client.delete( key )
+        true
+      end
+
+      # See Hoodoo::TransientStore::Base#close for details.
+      #
+      def close
+        @client.close()
+      end
+
+    private
+
+      # Connect to Memcached if possible and return the connected Dalli client
+      # instance, else raise an exception.
+      #
+      # +host+:: Connection URI, e.g. <tt>localhost:11211</tt>.
+      #
+      def connect_to_memcached( host, namespace )
+        exception = nil
+        stats     = nil
+        client    = nil
+
+        begin
+          client = ::Dalli::Client.new(
+            host,
+            {
+              :compress   => false,
+              :serializer => JSON,
+              :namespace  => namespace
+            }
+          )
+
+          stats = client.stats()
+
+        rescue => e
+          exception = e
+
+        end
+
+        if stats.nil?
+          if exception.nil?
+            raise "Hoodoo::TransientStore::Memcached: Did not get back meaningful data from Memcached at '#{ host }'"
+          else
+            raise "Hoodoo::TransientStore::Memcached: Cannot connect to Memcached at '#{ host }': #{ exception.to_s }"
+          end
+        else
+          return client
+        end
+      end
+
+    end
+
+    Hoodoo::TransientStore.register(
+      as:    :memcached,
+      using: Hoodoo::TransientStore::Memcached
+    ) if defined?( ::Dalli )
+
+  end
+end