hoodoo 1.13.0 → 1.14.0

data/lib/hoodoo/transient_store/transient_store/memcached_redis_mirror.rb

@@ -0,0 +1,181 @@
+########################################################################
+# File::    memcached_redis_mirror.rb
+# (C)::     Loyalty New Zealand 2017
+#
+# Purpose:: Hoodoo::TransientStore plugin supporting storage into both
+#           Memcached and Redis simultaneously.
+# ----------------------------------------------------------------------
+#           01-Feb-2017 (ADH): Created.
+########################################################################
+
+module Hoodoo
+  class TransientStore
+
+    # Hoodoo::TransientStore plugin supporting storage into both
+    # {Memcached}[https://memcached.org] and {Redis}[https://redis.io]
+    # simultaneously.
+    #
+    # The implementation uses Hoodoo::TransientStore::Memcached and
+    # Hoodoo::TransientStore::Redis to talk to the two storage engines.
+    #
+    # When looking up data with #get, the requested item must be found in both
+    # storage engines. If it is found in only one, the other one is deleted to
+    # keep maximum pool space available in both and +nil+ will be returned for
+    # the lookup.
+    #
+    # Note unusual requirements for the connection URI data provided to the
+    # #initialize call.
+    #
+    # The mirroring storage engine plug-in is useful if migrating from one of
+    # these engines to another without invalidating data present in the one
+    # from which you are migrating away. Change to using the mirrored storage
+    # engine for as long as the maximum item expiry period in the old engine,
+    # then once you know all old engine items must have been expired, cut over
+    # to just the new engine.
+    #
+    class MemcachedRedisMirror < Hoodoo::TransientStore::Base
+
+      # Command an instance to allow #get to return values which are only found
+      # in one or the other storage engine, or in both.
+      #
+      # Permitted values are:
+      #
+      # +:memcached+:: Only Memcached needs to have a value for a key
+      # +:redis+::     Only Redis needs to have a value for a key
+      # +both+::       Both engines must have the key
+      #
+      # This is useful in migration scenarios where moving from Memcached to
+      # Redis or vice versa. If wishing to be able to still read old data only
+      # in Memcached, set +:memcached+; else +:redis+. That way, data only in
+      # the old engine but not yet in the new is still considered valid and
+      # read back. For true mirroring which requires both stores to have the
+      # value, use +:both+.
+      #
+      # The default is +:both+.
+      #
+      attr_accessor :get_keys_from
+
+      # See Hoodoo::TransientStore::Base::new for details.
+      #
+      # Do not instantiate this class directly. Use
+      # Hoodoo::TransientStore::new.
+      #
+      # The +storage_host_uri+ parameter is necessarily unusual here. It must
+      # be _a Hash_ with Symbol keys +:memcached+ and +:redis+, those values
+      # giving the actual storage engine host URI for the respective engines.
+      # For example, to connect to locally running engines configured on their
+      # default ports, pass this Hash in +storage_host_uri+:
+      #
+      #     {
+      #       :memcached => 'localhost:11211',
+      #       :redis     => 'redis://localhost:6379'
+      #     }
+      #
+      # See Hoodoo::TransientStore::Memcached::new and
+      # Hoodoo::TransientStore::Redis::new for details of connection URI
+      # requirements for those engines.
+      #
+      # The value of the +namespace+ parameter applies equally to both engines.
+      #
+      def initialize( storage_host_uri:, namespace: )
+        super # Pass all arguments through -> *not* 'super()'
+
+        unless storage_host_uri.is_a?( Hash ) &&
+               storage_host_uri.has_key?( :memcached ) &&
+               storage_host_uri.has_key?( :redis )
+          raise 'Hoodoo::TransientStore::MemcachedRedisMirror: Bad storage host URI data passed to constructor'
+        end
+
+        @get_keys_from   = :both
+        @memcached_store = Hoodoo::TransientStore::Memcached.new( storage_host_uri: storage_host_uri[ :memcached ], namespace: namespace )
+        @redis_store     =     Hoodoo::TransientStore::Redis.new( storage_host_uri: storage_host_uri[ :redis     ], namespace: namespace )
+      end
+
+      # See Hoodoo::TransientStore::Base#set for details.
+      #
+      def set( key:, payload:, maximum_lifespan: )
+        memcached_result = @memcached_store.set( key: key, payload: payload, maximum_lifespan: maximum_lifespan )
+        redis_result     =     @redis_store.set( key: key, payload: payload, maximum_lifespan: maximum_lifespan )
+
+        return memcached_result && redis_result
+      end
+
+      # See Hoodoo::TransientStore::Base#get for details.
+      #
+      # The requested item must be found in both Memcached and Redis. If it is
+      # found in only one, the other one is deleted to keep maximum pool space
+      # available in and +nil+ will be returned.
+      #
+      # If #get_keys_from is configured for +:both+ and the data for some
+      # reason has ended up differing in the two stores - most likely because
+      # something modified just one of them (perhaps there is outdated code
+      # kicking around which is writing to just one) - then the Memcached copy
+      # will "win" and the Redis value will be ignored.
+      #
+      def get( key: )
+        case @get_keys_from
+          when :both
+            memcached_result = @memcached_store.get( key: key )
+            redis_result     =     @redis_store.get( key: key )
+
+            if @get_keys_from == :both && ( memcached_result.nil? || redis_result.nil? )
+              delete( key: key )
+              return nil
+            else
+              return memcached_result
+            end
+
+          when :memcached
+            return @memcached_store.get( key: key )
+
+          when :redis
+            return @redis_store.get( key: key )
+
+          else
+            raise "Hoodoo::TransientStore::Base\#get: Invalid prior value given in \#get_keys_from= of '#{ @get_keys_from.inspect }' - only ':both', ':memcached' or ':redis' are allowed"
+
+        end
+      end
+
+      # See Hoodoo::TransientStore::Base#delete for details.
+      #
+      def delete( key: )
+        exception = nil
+
+        begin
+          @memcached_store.delete( key: key )
+        rescue => e
+          exception = e
+        end
+
+        # But allow Redis delete to still be attempted...
+
+        begin
+          @redis_store.delete( key: key )
+        rescue => e
+          exception ||= e
+        end
+
+        if exception.nil?
+          return true
+        else
+          raise exception
+        end
+      end
+
+      # See Hoodoo::TransientStore::Base#close for details.
+      #
+      def close
+        @memcached_store.close() rescue nil # Rescue so that Redis "close()" is still attempted.
+            @redis_store.close()
+      end
+
+    end
+
+    Hoodoo::TransientStore.register(
+      as:    :memcached_redis_mirror,
+      using: Hoodoo::TransientStore::MemcachedRedisMirror
+    ) if ( defined?( ::Redis ) && defined?( ::Dalli ) )
+
+  end
+end

data/lib/hoodoo/transient_store/transient_store/redis.rb

@@ -0,0 +1,126 @@
+########################################################################
+# File::    redis.rb
+# (C)::     Loyalty New Zealand 2017
+#
+# Purpose:: Hoodoo::TransientStore plugin supporting Redis.
+# ----------------------------------------------------------------------
+#           01-Feb-2017 (ADH): Created.
+########################################################################
+
+begin
+  require 'json'
+  require 'redis'
+rescue LoadError
+end
+
+module Hoodoo
+  class TransientStore
+
+    # Hoodoo::TransientStore plugin supporting {Redis}[https://redis.io]. The
+    # {redis-rb gem}[https://github.com/redis/redis-rb] is used for server
+    # communication.
+    #
+    class Redis < Hoodoo::TransientStore::Base
+
+      # See Hoodoo::TransientStore::Base::new for details.
+      #
+      # Do not instantiate this class directly. Use
+      # Hoodoo::TransientStore::new.
+      #
+      # The {redis-rb gem}[https://github.com/redis/redis-rb] is used to talk
+      # to {Redis}[https://redis.io] and requires connection UIRs with a
+      # +redis+ protocol, such as <tt>redis://localhost:6379</tt>.
+      #
+      # TCP keep-alive is enabled for the server connection.
+      #
+      def initialize( storage_host_uri:, namespace: )
+        super # Pass all arguments through -> *not* 'super()'
+        @client = connect_to_redis( storage_host_uri )
+      end
+
+      # See Hoodoo::TransientStore::Base#set for details.
+      #
+      # The payload is encoded into JSON for storage and automatically decoded
+      # by #get; so callers don't need to do marshalling to Strings themselves.
+      #
+      def set( key:, payload:, maximum_lifespan: )
+        nk = namespaced_key( key )
+
+        @client[ nk ] = JSON.fast_generate( payload )
+        @client.expire( nk, maximum_lifespan )
+
+        true
+      end
+
+      # See Hoodoo::TransientStore::Base#get for details.
+      #
+      def get( key: )
+        result = @client[ namespaced_key( key ) ]
+        return result.nil? ? nil : ( JSON.parse( result ) rescue nil )
+      end
+
+      # See Hoodoo::TransientStore::Base#delete for details.
+      #
+      def delete( key: )
+        @client.del( namespaced_key( key ) )
+        true
+      end
+
+      # See Hoodoo::TransientStore::Base#close for details.
+      #
+      def close
+        @client.quit()
+      end
+
+    private
+
+      # Given a simple key to Redis data (expressed as a String), return a
+      # namespaced version by adding the configured namespace as a prefix.
+      #
+      def namespaced_key( key )
+        @namespace + key
+      end
+
+      # Connect to Redis if possible and return the connected Redis client
+      # instance, else raise an exception.
+      #
+      # +host+:: Connection URI, e.g. <tt>localhost:11211</tt>.
+      #
+      def connect_to_redis( host )
+        exception = nil
+        info      = nil
+        client    = nil
+
+        begin
+          client = ::Redis.new(
+            :url           => host,
+            :tcp_keepalive => 1
+          )
+
+          info = client.info( 'CPU' )
+
+        rescue => e
+          exception = e
+
+        end
+
+        if info.nil?
+          if exception.nil?
+            raise "Hoodoo::TransientStore::Redis: Did not get back meaningful data from Redis at '#{ host }'"
+          else
+            raise "Hoodoo::TransientStore::Redis: Cannot connect to Redis at '#{ host }': #{ exception.to_s }"
+          end
+        else
+          return client
+        end
+      end
+
+    end
+
+    Hoodoo::TransientStore.register(
+      as:    :redis,
+      using: Hoodoo::TransientStore::Redis
+    ) if defined?( ::Redis )
+
+  end
+end

data/lib/hoodoo/version.rb

@@ -12,6 +12,6 @@ module Hoodoo
   # The Hoodoo gem version. If this changes, ensure that the date in
   # "hoodoo.gemspec" is correct and run "bundle install" (or "update").
   #
-  VERSION = '1.13.0'
+  VERSION = '1.14.0'
 
 end

data/spec/active/active_record/support_spec.rb

@@ -212,9 +212,7 @@ describe Hoodoo::ActiveRecord::Support do
           expect( manual_scope ).to include( "\"r_spec_full_scope_for_test_subclasses\".\"foo\" = '#{ @test_scoping_value }'" )
         end
 
-        pending 'translated' do
-          raise "Scope verification for '\#translated'"
-        end
+        pending 'translated' # Scope verification for '#translated'
 
         # In this last one, we actually check full_scope_for; activations are in
         # the subclass so we drive the non-context versions directly to verify
@@ -250,9 +248,7 @@ describe Hoodoo::ActiveRecord::Support do
         expect( manual_scope ).to include( "\"rspec_full_scope_for_test_base_with_directives_custom\".\"bar\" = '#{ @test_scoping_value }'" )
       end
 
-      pending 'translated' do
-        raise "Scope verification for '\#translated'"
-      end
+      pending 'translated' # Scope verification for '#translated'
 
       it 'yields the same SQL' do
         auto_scope   = described_class.full_scope_for( RSpecFullScopeForTestBaseSubclassWithoutOverrides, @context ).to_sql()
@@ -281,9 +277,7 @@ describe Hoodoo::ActiveRecord::Support do
           expect( manual_scope ).to include( "\"r_spec_full_scope_for_manually_dateds\".\"baz\" = '#{ @test_scoping_value }'" )
         end
 
-        pending 'translated' do
-          raise "Scope verification for '\#translated'"
-        end
+        pending 'translated' # Scope verification for '#translated'
 
         it 'everything' do
           auto_scope   = described_class.full_scope_for( RSpecFullScopeForManuallyDated, @context ).to_sql()

data/spec/active/active_record/translated_spec.rb

@@ -2,10 +2,9 @@ require 'spec_helper'
 require 'active_record'
 
 describe Hoodoo::ActiveRecord::Translated do
-  pending 'is tested' do
-
-    # For RCov only
+  pending 'is tested' # Replace with real tests!
 
+  it 'temporarily addresses RCov coverage' do
     class Test
       include Hoodoo::ActiveRecord::Translated
       def self.all; end
@@ -13,7 +12,5 @@ describe Hoodoo::ActiveRecord::Translated do
 
     ignored = true
     Test.translated( ignored )
-
-    raise "Replace with real test"
   end
 end

data/spec/logger/writers/file_writer_spec.rb

@@ -9,10 +9,7 @@ describe Hoodoo::Logger::FileWriter do
   end
 
   after :all do
-    begin
-      File.unlink( @temp_path )
-    rescue
-    end
+    File.unlink( @temp_path ) rescue nil
   end
 
   it 'writes to files' do

data/spec/logger/writers/stream_writer_spec.rb

@@ -9,15 +9,8 @@ describe Hoodoo::Logger::StreamWriter do
   end
 
   after :all do
-    begin
-      @stream.close
-    rescue
-    end
-
-    begin
-      File.unlink( @temp_path )
-    rescue
-    end
+    @stream.close             rescue nil
+    File.unlink( @temp_path ) rescue nil
   end
 
   it 'writes to streams' do

data/spec/services/middleware/middleware_logging_spec.rb

@@ -70,10 +70,7 @@ describe Hoodoo::Services::Middleware do
     force_logging_to( 'test' )
     Hoodoo::Services::Middleware.class_variable_set( '@@environment', @old_env )
     Hoodoo::Services::Middleware.class_variable_set( '@@logger', @old_logger )
-    begin
-      Hoodoo::Services::Middleware.remove_class_variable( '@@alchemy' )
-    rescue
-    end
+    Hoodoo::Services::Middleware.remove_class_variable( '@@alchemy' ) rescue nil
   end
 
   context 'custom loggers' do

data/spec/services/middleware/middleware_permissions_spec.rb

@@ -343,14 +343,14 @@ describe Hoodoo::Services::Middleware do
       Hoodoo::Services::Permissions::ALLOW
     )
 
-    Hoodoo::Services::Session::MockDalliClient.reset()
+    Hoodoo::TransientStore::Mocks::DalliClient.reset()
 
     result = @session.save_to_memcached
     raise "Can't save to mock Memcached (result = #{result})" unless result == :ok
   end
 
   after :each do
-    Hoodoo::Services::Session::MockDalliClient.reset()
+    Hoodoo::TransientStore::Mocks::DalliClient.reset()
   end
 
   ############################################################################

data/spec/services/services/interface_spec.rb

@@ -259,9 +259,9 @@ describe Hoodoo::Services::Interface do
       it 'should complain about incorrect types' do
         expect {
           Hoodoo::Services::Interface::ToListDSL.new( Hoodoo::Services::Interface::ToList.new ) do
-            default 42
+            default( { :some => :hash } )
           end
-        }.to raise_error(RuntimeError, "Hoodoo::Services::Interface::ToListDSL\#default requires a String or Symbol - got 'Fixnum'")
+        }.to raise_error(RuntimeError, "Hoodoo::Services::Interface::ToListDSL\#default requires a String or Symbol - got 'Hash'")
       end
     end