hoodoo 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. checksums.yaml +4 -4
  2. data/lib/hoodoo.rb +1 -0
  3. data/lib/hoodoo/services/services/session.rb +8 -104
  4. data/lib/hoodoo/transient_store.rb +19 -0
  5. data/lib/hoodoo/transient_store/mocks/dalli_client.rb +148 -0
  6. data/lib/hoodoo/transient_store/mocks/redis.rb +138 -0
  7. data/lib/hoodoo/transient_store/transient_store.rb +344 -0
  8. data/lib/hoodoo/transient_store/transient_store/base.rb +81 -0
  9. data/lib/hoodoo/transient_store/transient_store/memcached.rb +116 -0
  10. data/lib/hoodoo/transient_store/transient_store/memcached_redis_mirror.rb +181 -0
  11. data/lib/hoodoo/transient_store/transient_store/redis.rb +126 -0
  12. data/lib/hoodoo/version.rb +1 -1
  13. data/spec/active/active_record/support_spec.rb +3 -9
  14. data/spec/active/active_record/translated_spec.rb +2 -5
  15. data/spec/logger/writers/file_writer_spec.rb +1 -4
  16. data/spec/logger/writers/stream_writer_spec.rb +2 -9
  17. data/spec/services/middleware/middleware_logging_spec.rb +1 -4
  18. data/spec/services/middleware/middleware_permissions_spec.rb +2 -2
  19. data/spec/services/services/interface_spec.rb +2 -2
  20. data/spec/services/services/session_spec.rb +26 -19
  21. data/spec/transient_store/transient_store/base_spec.rb +52 -0
  22. data/spec/transient_store/transient_store/memcached_redis_mirror_spec.rb +380 -0
  23. data/spec/transient_store/transient_store/memcached_spec.rb +244 -0
  24. data/spec/transient_store/transient_store/mocks/dalli_client_spec.rb +44 -0
  25. data/spec/transient_store/transient_store/mocks/redis_spec.rb +28 -0
  26. data/spec/transient_store/transient_store/redis_spec.rb +242 -0
  27. data/spec/transient_store/transient_store_spec.rb +448 -0
  28. metadata +31 -9
@@ -0,0 +1,181 @@
1
+ ########################################################################
2
+ # File:: memcached_redis_mirror.rb
3
+ # (C):: Loyalty New Zealand 2017
4
+ #
5
+ # Purpose:: Hoodoo::TransientStore plugin supporting storage into both
6
+ # Memcached and Redis simultaneously.
7
+ # ----------------------------------------------------------------------
8
+ # 01-Feb-2017 (ADH): Created.
9
+ ########################################################################
10
+
11
+ module Hoodoo
12
+ class TransientStore
13
+
14
+ # Hoodoo::TransientStore plugin supporting storage into both
15
+ # {Memcached}[https://memcached.org] and {Redis}[https://redis.io]
16
+ # simultaneously.
17
+ #
18
+ # The implementation uses Hoodoo::TransientStore::Memcached and
19
+ # Hoodoo::TransientStore::Redis to talk to the two storage engines.
20
+ #
21
+ # When looking up data with #get, the requested item must be found in both
22
+ # storage engines. If it is found in only one, the other one is deleted to
23
+ # keep maximum pool space available in both and +nil+ will be returned for
24
+ # the lookup.
25
+ #
26
+ # Note unusual requirements for the connection URI data provided to the
27
+ # #initialize call.
28
+ #
29
+ # The mirroring storage engine plug-in is useful if migrating from one of
30
+ # these engines to another without invalidating data present in the one
31
+ # from which you are migrating away. Change to using the mirrored storage
32
+ # engine for as long as the maximum item expiry period in the old engine,
33
+ # then once you know all old engine items must have been expired, cut over
34
+ # to just the new engine.
35
+ #
36
+ class MemcachedRedisMirror < Hoodoo::TransientStore::Base
37
+
38
+ # Command an instance to allow #get to return values which are only found
39
+ # in one or the other storage engine, or in both.
40
+ #
41
+ # Permitted values are:
42
+ #
43
+ # +:memcached+:: Only Memcached needs to have a value for a key
44
+ # +:redis+:: Only Redis needs to have a value for a key
45
+ # +both+:: Both engines must have the key
46
+ #
47
+ # This is useful in migration scenarios where moving from Memcached to
48
+ # Redis or vice versa. If wishing to be able to still read old data only
49
+ # in Memcached, set +:memcached+; else +:redis+. That way, data only in
50
+ # the old engine but not yet in the new is still considered valid and
51
+ # read back. For true mirroring which requires both stores to have the
52
+ # value, use +:both+.
53
+ #
54
+ # The default is +:both+.
55
+ #
56
+ attr_accessor :get_keys_from
57
+
58
+ # See Hoodoo::TransientStore::Base::new for details.
59
+ #
60
+ # Do not instantiate this class directly. Use
61
+ # Hoodoo::TransientStore::new.
62
+ #
63
+ # The +storage_host_uri+ parameter is necessarily unusual here. It must
64
+ # be _a Hash_ with Symbol keys +:memcached+ and +:redis+, those values
65
+ # giving the actual storage engine host URI for the respective engines.
66
+ # For example, to connect to locally running engines configured on their
67
+ # default ports, pass this Hash in +storage_host_uri+:
68
+ #
69
+ # {
70
+ # :memcached => 'localhost:11211',
71
+ # :redis => 'redis://localhost:6379'
72
+ # }
73
+ #
74
+ # See Hoodoo::TransientStore::Memcached::new and
75
+ # Hoodoo::TransientStore::Redis::new for details of connection URI
76
+ # requirements for those engines.
77
+ #
78
+ # The value of the +namespace+ parameter applies equally to both engines.
79
+ #
80
+ def initialize( storage_host_uri:, namespace: )
81
+ super # Pass all arguments through -> *not* 'super()'
82
+
83
+ unless storage_host_uri.is_a?( Hash ) &&
84
+ storage_host_uri.has_key?( :memcached ) &&
85
+ storage_host_uri.has_key?( :redis )
86
+ raise 'Hoodoo::TransientStore::MemcachedRedisMirror: Bad storage host URI data passed to constructor'
87
+ end
88
+
89
+ @get_keys_from = :both
90
+ @memcached_store = Hoodoo::TransientStore::Memcached.new( storage_host_uri: storage_host_uri[ :memcached ], namespace: namespace )
91
+ @redis_store = Hoodoo::TransientStore::Redis.new( storage_host_uri: storage_host_uri[ :redis ], namespace: namespace )
92
+ end
93
+
94
+ # See Hoodoo::TransientStore::Base#set for details.
95
+ #
96
+ def set( key:, payload:, maximum_lifespan: )
97
+ memcached_result = @memcached_store.set( key: key, payload: payload, maximum_lifespan: maximum_lifespan )
98
+ redis_result = @redis_store.set( key: key, payload: payload, maximum_lifespan: maximum_lifespan )
99
+
100
+ return memcached_result && redis_result
101
+ end
102
+
103
+ # See Hoodoo::TransientStore::Base#get for details.
104
+ #
105
+ # The requested item must be found in both Memcached and Redis. If it is
106
+ # found in only one, the other one is deleted to keep maximum pool space
107
+ # available in and +nil+ will be returned.
108
+ #
109
+ # If #get_keys_from is configured for +:both+ and the data for some
110
+ # reason has ended up differing in the two stores - most likely because
111
+ # something modified just one of them (perhaps there is outdated code
112
+ # kicking around which is writing to just one) - then the Memcached copy
113
+ # will "win" and the Redis value will be ignored.
114
+ #
115
+ def get( key: )
116
+ case @get_keys_from
117
+ when :both
118
+ memcached_result = @memcached_store.get( key: key )
119
+ redis_result = @redis_store.get( key: key )
120
+
121
+ if @get_keys_from == :both && ( memcached_result.nil? || redis_result.nil? )
122
+ delete( key: key )
123
+ return nil
124
+ else
125
+ return memcached_result
126
+ end
127
+
128
+ when :memcached
129
+ return @memcached_store.get( key: key )
130
+
131
+ when :redis
132
+ return @redis_store.get( key: key )
133
+
134
+ else
135
+ raise "Hoodoo::TransientStore::Base\#get: Invalid prior value given in \#get_keys_from= of '#{ @get_keys_from.inspect }' - only ':both', ':memcached' or ':redis' are allowed"
136
+
137
+ end
138
+ end
139
+
140
+ # See Hoodoo::TransientStore::Base#delete for details.
141
+ #
142
+ def delete( key: )
143
+ exception = nil
144
+
145
+ begin
146
+ @memcached_store.delete( key: key )
147
+ rescue => e
148
+ exception = e
149
+ end
150
+
151
+ # But allow Redis delete to still be attempted...
152
+
153
+ begin
154
+ @redis_store.delete( key: key )
155
+ rescue => e
156
+ exception ||= e
157
+ end
158
+
159
+ if exception.nil?
160
+ return true
161
+ else
162
+ raise exception
163
+ end
164
+ end
165
+
166
+ # See Hoodoo::TransientStore::Base#close for details.
167
+ #
168
+ def close
169
+ @memcached_store.close() rescue nil # Rescue so that Redis "close()" is still attempted.
170
+ @redis_store.close()
171
+ end
172
+
173
+ end
174
+
175
+ Hoodoo::TransientStore.register(
176
+ as: :memcached_redis_mirror,
177
+ using: Hoodoo::TransientStore::MemcachedRedisMirror
178
+ ) if ( defined?( ::Redis ) && defined?( ::Dalli ) )
179
+
180
+ end
181
+ end
@@ -0,0 +1,126 @@
1
+ ########################################################################
2
+ # File:: redis.rb
3
+ # (C):: Loyalty New Zealand 2017
4
+ #
5
+ # Purpose:: Hoodoo::TransientStore plugin supporting Redis.
6
+ # ----------------------------------------------------------------------
7
+ # 01-Feb-2017 (ADH): Created.
8
+ ########################################################################
9
+
10
+ begin
11
+ require 'json'
12
+ require 'redis'
13
+ rescue LoadError
14
+ end
15
+
16
+ module Hoodoo
17
+ class TransientStore
18
+
19
+ # Hoodoo::TransientStore plugin supporting {Redis}[https://redis.io]. The
20
+ # {redis-rb gem}[https://github.com/redis/redis-rb] is used for server
21
+ # communication.
22
+ #
23
+ class Redis < Hoodoo::TransientStore::Base
24
+
25
+ # See Hoodoo::TransientStore::Base::new for details.
26
+ #
27
+ # Do not instantiate this class directly. Use
28
+ # Hoodoo::TransientStore::new.
29
+ #
30
+ # The {redis-rb gem}[https://github.com/redis/redis-rb] is used to talk
31
+ # to {Redis}[https://redis.io] and requires connection UIRs with a
32
+ # +redis+ protocol, such as <tt>redis://localhost:6379</tt>.
33
+ #
34
+ # TCP keep-alive is enabled for the server connection.
35
+ #
36
+ def initialize( storage_host_uri:, namespace: )
37
+ super # Pass all arguments through -> *not* 'super()'
38
+ @client = connect_to_redis( storage_host_uri )
39
+ end
40
+
41
+ # See Hoodoo::TransientStore::Base#set for details.
42
+ #
43
+ # The payload is encoded into JSON for storage and automatically decoded
44
+ # by #get; so callers don't need to do marshalling to Strings themselves.
45
+ #
46
+ def set( key:, payload:, maximum_lifespan: )
47
+ nk = namespaced_key( key )
48
+
49
+ @client[ nk ] = JSON.fast_generate( payload )
50
+ @client.expire( nk, maximum_lifespan )
51
+
52
+ true
53
+ end
54
+
55
+ # See Hoodoo::TransientStore::Base#get for details.
56
+ #
57
+ def get( key: )
58
+ result = @client[ namespaced_key( key ) ]
59
+ return result.nil? ? nil : ( JSON.parse( result ) rescue nil )
60
+ end
61
+
62
+ # See Hoodoo::TransientStore::Base#delete for details.
63
+ #
64
+ def delete( key: )
65
+ @client.del( namespaced_key( key ) )
66
+ true
67
+ end
68
+
69
+ # See Hoodoo::TransientStore::Base#close for details.
70
+ #
71
+ def close
72
+ @client.quit()
73
+ end
74
+
75
+ private
76
+
77
+ # Given a simple key to Redis data (expressed as a String), return a
78
+ # namespaced version by adding the configured namespace as a prefix.
79
+ #
80
+ def namespaced_key( key )
81
+ @namespace + key
82
+ end
83
+
84
+ # Connect to Redis if possible and return the connected Redis client
85
+ # instance, else raise an exception.
86
+ #
87
+ # +host+:: Connection URI, e.g. <tt>localhost:11211</tt>.
88
+ #
89
+ def connect_to_redis( host )
90
+ exception = nil
91
+ info = nil
92
+ client = nil
93
+
94
+ begin
95
+ client = ::Redis.new(
96
+ :url => host,
97
+ :tcp_keepalive => 1
98
+ )
99
+
100
+ info = client.info( 'CPU' )
101
+
102
+ rescue => e
103
+ exception = e
104
+
105
+ end
106
+
107
+ if info.nil?
108
+ if exception.nil?
109
+ raise "Hoodoo::TransientStore::Redis: Did not get back meaningful data from Redis at '#{ host }'"
110
+ else
111
+ raise "Hoodoo::TransientStore::Redis: Cannot connect to Redis at '#{ host }': #{ exception.to_s }"
112
+ end
113
+ else
114
+ return client
115
+ end
116
+ end
117
+
118
+ end
119
+
120
+ Hoodoo::TransientStore.register(
121
+ as: :redis,
122
+ using: Hoodoo::TransientStore::Redis
123
+ ) if defined?( ::Redis )
124
+
125
+ end
126
+ end
@@ -12,6 +12,6 @@ module Hoodoo
12
12
  # The Hoodoo gem version. If this changes, ensure that the date in
13
13
  # "hoodoo.gemspec" is correct and run "bundle install" (or "update").
14
14
  #
15
- VERSION = '1.13.0'
15
+ VERSION = '1.14.0'
16
16
 
17
17
  end
@@ -212,9 +212,7 @@ describe Hoodoo::ActiveRecord::Support do
212
212
  expect( manual_scope ).to include( "\"r_spec_full_scope_for_test_subclasses\".\"foo\" = '#{ @test_scoping_value }'" )
213
213
  end
214
214
 
215
- pending 'translated' do
216
- raise "Scope verification for '\#translated'"
217
- end
215
+ pending 'translated' # Scope verification for '#translated'
218
216
 
219
217
  # In this last one, we actually check full_scope_for; activations are in
220
218
  # the subclass so we drive the non-context versions directly to verify
@@ -250,9 +248,7 @@ describe Hoodoo::ActiveRecord::Support do
250
248
  expect( manual_scope ).to include( "\"rspec_full_scope_for_test_base_with_directives_custom\".\"bar\" = '#{ @test_scoping_value }'" )
251
249
  end
252
250
 
253
- pending 'translated' do
254
- raise "Scope verification for '\#translated'"
255
- end
251
+ pending 'translated' # Scope verification for '#translated'
256
252
 
257
253
  it 'yields the same SQL' do
258
254
  auto_scope = described_class.full_scope_for( RSpecFullScopeForTestBaseSubclassWithoutOverrides, @context ).to_sql()
@@ -281,9 +277,7 @@ describe Hoodoo::ActiveRecord::Support do
281
277
  expect( manual_scope ).to include( "\"r_spec_full_scope_for_manually_dateds\".\"baz\" = '#{ @test_scoping_value }'" )
282
278
  end
283
279
 
284
- pending 'translated' do
285
- raise "Scope verification for '\#translated'"
286
- end
280
+ pending 'translated' # Scope verification for '#translated'
287
281
 
288
282
  it 'everything' do
289
283
  auto_scope = described_class.full_scope_for( RSpecFullScopeForManuallyDated, @context ).to_sql()
@@ -2,10 +2,9 @@ require 'spec_helper'
2
2
  require 'active_record'
3
3
 
4
4
  describe Hoodoo::ActiveRecord::Translated do
5
- pending 'is tested' do
6
-
7
- # For RCov only
5
+ pending 'is tested' # Replace with real tests!
8
6
 
7
+ it 'temporarily addresses RCov coverage' do
9
8
  class Test
10
9
  include Hoodoo::ActiveRecord::Translated
11
10
  def self.all; end
@@ -13,7 +12,5 @@ describe Hoodoo::ActiveRecord::Translated do
13
12
 
14
13
  ignored = true
15
14
  Test.translated( ignored )
16
-
17
- raise "Replace with real test"
18
15
  end
19
16
  end
@@ -9,10 +9,7 @@ describe Hoodoo::Logger::FileWriter do
9
9
  end
10
10
 
11
11
  after :all do
12
- begin
13
- File.unlink( @temp_path )
14
- rescue
15
- end
12
+ File.unlink( @temp_path ) rescue nil
16
13
  end
17
14
 
18
15
  it 'writes to files' do
@@ -9,15 +9,8 @@ describe Hoodoo::Logger::StreamWriter do
9
9
  end
10
10
 
11
11
  after :all do
12
- begin
13
- @stream.close
14
- rescue
15
- end
16
-
17
- begin
18
- File.unlink( @temp_path )
19
- rescue
20
- end
12
+ @stream.close rescue nil
13
+ File.unlink( @temp_path ) rescue nil
21
14
  end
22
15
 
23
16
  it 'writes to streams' do
@@ -70,10 +70,7 @@ describe Hoodoo::Services::Middleware do
70
70
  force_logging_to( 'test' )
71
71
  Hoodoo::Services::Middleware.class_variable_set( '@@environment', @old_env )
72
72
  Hoodoo::Services::Middleware.class_variable_set( '@@logger', @old_logger )
73
- begin
74
- Hoodoo::Services::Middleware.remove_class_variable( '@@alchemy' )
75
- rescue
76
- end
73
+ Hoodoo::Services::Middleware.remove_class_variable( '@@alchemy' ) rescue nil
77
74
  end
78
75
 
79
76
  context 'custom loggers' do
@@ -343,14 +343,14 @@ describe Hoodoo::Services::Middleware do
343
343
  Hoodoo::Services::Permissions::ALLOW
344
344
  )
345
345
 
346
- Hoodoo::Services::Session::MockDalliClient.reset()
346
+ Hoodoo::TransientStore::Mocks::DalliClient.reset()
347
347
 
348
348
  result = @session.save_to_memcached
349
349
  raise "Can't save to mock Memcached (result = #{result})" unless result == :ok
350
350
  end
351
351
 
352
352
  after :each do
353
- Hoodoo::Services::Session::MockDalliClient.reset()
353
+ Hoodoo::TransientStore::Mocks::DalliClient.reset()
354
354
  end
355
355
 
356
356
  ############################################################################
@@ -259,9 +259,9 @@ describe Hoodoo::Services::Interface do
259
259
  it 'should complain about incorrect types' do
260
260
  expect {
261
261
  Hoodoo::Services::Interface::ToListDSL.new( Hoodoo::Services::Interface::ToList.new ) do
262
- default 42
262
+ default( { :some => :hash } )
263
263
  end
264
- }.to raise_error(RuntimeError, "Hoodoo::Services::Interface::ToListDSL\#default requires a String or Symbol - got 'Fixnum'")
264
+ }.to raise_error(RuntimeError, "Hoodoo::Services::Interface::ToListDSL\#default requires a String or Symbol - got 'Hash'")
265
265
  end
266
266
  end
267
267