hoodoo 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hoodoo.rb +1 -0
- data/lib/hoodoo/services/services/session.rb +8 -104
- data/lib/hoodoo/transient_store.rb +19 -0
- data/lib/hoodoo/transient_store/mocks/dalli_client.rb +148 -0
- data/lib/hoodoo/transient_store/mocks/redis.rb +138 -0
- data/lib/hoodoo/transient_store/transient_store.rb +344 -0
- data/lib/hoodoo/transient_store/transient_store/base.rb +81 -0
- data/lib/hoodoo/transient_store/transient_store/memcached.rb +116 -0
- data/lib/hoodoo/transient_store/transient_store/memcached_redis_mirror.rb +181 -0
- data/lib/hoodoo/transient_store/transient_store/redis.rb +126 -0
- data/lib/hoodoo/version.rb +1 -1
- data/spec/active/active_record/support_spec.rb +3 -9
- data/spec/active/active_record/translated_spec.rb +2 -5
- data/spec/logger/writers/file_writer_spec.rb +1 -4
- data/spec/logger/writers/stream_writer_spec.rb +2 -9
- data/spec/services/middleware/middleware_logging_spec.rb +1 -4
- data/spec/services/middleware/middleware_permissions_spec.rb +2 -2
- data/spec/services/services/interface_spec.rb +2 -2
- data/spec/services/services/session_spec.rb +26 -19
- data/spec/transient_store/transient_store/base_spec.rb +52 -0
- data/spec/transient_store/transient_store/memcached_redis_mirror_spec.rb +380 -0
- data/spec/transient_store/transient_store/memcached_spec.rb +244 -0
- data/spec/transient_store/transient_store/mocks/dalli_client_spec.rb +44 -0
- data/spec/transient_store/transient_store/mocks/redis_spec.rb +28 -0
- data/spec/transient_store/transient_store/redis_spec.rb +242 -0
- data/spec/transient_store/transient_store_spec.rb +448 -0
- metadata +31 -9
@@ -0,0 +1,181 @@
|
|
1
|
+
########################################################################
|
2
|
+
# File:: memcached_redis_mirror.rb
|
3
|
+
# (C):: Loyalty New Zealand 2017
|
4
|
+
#
|
5
|
+
# Purpose:: Hoodoo::TransientStore plugin supporting storage into both
|
6
|
+
# Memcached and Redis simultaneously.
|
7
|
+
# ----------------------------------------------------------------------
|
8
|
+
# 01-Feb-2017 (ADH): Created.
|
9
|
+
########################################################################
|
10
|
+
|
11
|
+
module Hoodoo
|
12
|
+
class TransientStore
|
13
|
+
|
14
|
+
# Hoodoo::TransientStore plugin supporting storage into both
|
15
|
+
# {Memcached}[https://memcached.org] and {Redis}[https://redis.io]
|
16
|
+
# simultaneously.
|
17
|
+
#
|
18
|
+
# The implementation uses Hoodoo::TransientStore::Memcached and
|
19
|
+
# Hoodoo::TransientStore::Redis to talk to the two storage engines.
|
20
|
+
#
|
21
|
+
# When looking up data with #get, the requested item must be found in both
|
22
|
+
# storage engines. If it is found in only one, the other one is deleted to
|
23
|
+
# keep maximum pool space available in both and +nil+ will be returned for
|
24
|
+
# the lookup.
|
25
|
+
#
|
26
|
+
# Note unusual requirements for the connection URI data provided to the
|
27
|
+
# #initialize call.
|
28
|
+
#
|
29
|
+
# The mirroring storage engine plug-in is useful if migrating from one of
|
30
|
+
# these engines to another without invalidating data present in the one
|
31
|
+
# from which you are migrating away. Change to using the mirrored storage
|
32
|
+
# engine for as long as the maximum item expiry period in the old engine,
|
33
|
+
# then once you know all old engine items must have been expired, cut over
|
34
|
+
# to just the new engine.
|
35
|
+
#
|
36
|
+
class MemcachedRedisMirror < Hoodoo::TransientStore::Base
|
37
|
+
|
38
|
+
# Command an instance to allow #get to return values which are only found
|
39
|
+
# in one or the other storage engine, or in both.
|
40
|
+
#
|
41
|
+
# Permitted values are:
|
42
|
+
#
|
43
|
+
# +:memcached+:: Only Memcached needs to have a value for a key
|
44
|
+
# +:redis+:: Only Redis needs to have a value for a key
|
45
|
+
# +both+:: Both engines must have the key
|
46
|
+
#
|
47
|
+
# This is useful in migration scenarios where moving from Memcached to
|
48
|
+
# Redis or vice versa. If wishing to be able to still read old data only
|
49
|
+
# in Memcached, set +:memcached+; else +:redis+. That way, data only in
|
50
|
+
# the old engine but not yet in the new is still considered valid and
|
51
|
+
# read back. For true mirroring which requires both stores to have the
|
52
|
+
# value, use +:both+.
|
53
|
+
#
|
54
|
+
# The default is +:both+.
|
55
|
+
#
|
56
|
+
attr_accessor :get_keys_from
|
57
|
+
|
58
|
+
# See Hoodoo::TransientStore::Base::new for details.
|
59
|
+
#
|
60
|
+
# Do not instantiate this class directly. Use
|
61
|
+
# Hoodoo::TransientStore::new.
|
62
|
+
#
|
63
|
+
# The +storage_host_uri+ parameter is necessarily unusual here. It must
|
64
|
+
# be _a Hash_ with Symbol keys +:memcached+ and +:redis+, those values
|
65
|
+
# giving the actual storage engine host URI for the respective engines.
|
66
|
+
# For example, to connect to locally running engines configured on their
|
67
|
+
# default ports, pass this Hash in +storage_host_uri+:
|
68
|
+
#
|
69
|
+
# {
|
70
|
+
# :memcached => 'localhost:11211',
|
71
|
+
# :redis => 'redis://localhost:6379'
|
72
|
+
# }
|
73
|
+
#
|
74
|
+
# See Hoodoo::TransientStore::Memcached::new and
|
75
|
+
# Hoodoo::TransientStore::Redis::new for details of connection URI
|
76
|
+
# requirements for those engines.
|
77
|
+
#
|
78
|
+
# The value of the +namespace+ parameter applies equally to both engines.
|
79
|
+
#
|
80
|
+
def initialize( storage_host_uri:, namespace: )
|
81
|
+
super # Pass all arguments through -> *not* 'super()'
|
82
|
+
|
83
|
+
unless storage_host_uri.is_a?( Hash ) &&
|
84
|
+
storage_host_uri.has_key?( :memcached ) &&
|
85
|
+
storage_host_uri.has_key?( :redis )
|
86
|
+
raise 'Hoodoo::TransientStore::MemcachedRedisMirror: Bad storage host URI data passed to constructor'
|
87
|
+
end
|
88
|
+
|
89
|
+
@get_keys_from = :both
|
90
|
+
@memcached_store = Hoodoo::TransientStore::Memcached.new( storage_host_uri: storage_host_uri[ :memcached ], namespace: namespace )
|
91
|
+
@redis_store = Hoodoo::TransientStore::Redis.new( storage_host_uri: storage_host_uri[ :redis ], namespace: namespace )
|
92
|
+
end
|
93
|
+
|
94
|
+
# See Hoodoo::TransientStore::Base#set for details.
|
95
|
+
#
|
96
|
+
def set( key:, payload:, maximum_lifespan: )
|
97
|
+
memcached_result = @memcached_store.set( key: key, payload: payload, maximum_lifespan: maximum_lifespan )
|
98
|
+
redis_result = @redis_store.set( key: key, payload: payload, maximum_lifespan: maximum_lifespan )
|
99
|
+
|
100
|
+
return memcached_result && redis_result
|
101
|
+
end
|
102
|
+
|
103
|
+
# See Hoodoo::TransientStore::Base#get for details.
|
104
|
+
#
|
105
|
+
# The requested item must be found in both Memcached and Redis. If it is
|
106
|
+
# found in only one, the other one is deleted to keep maximum pool space
|
107
|
+
# available in and +nil+ will be returned.
|
108
|
+
#
|
109
|
+
# If #get_keys_from is configured for +:both+ and the data for some
|
110
|
+
# reason has ended up differing in the two stores - most likely because
|
111
|
+
# something modified just one of them (perhaps there is outdated code
|
112
|
+
# kicking around which is writing to just one) - then the Memcached copy
|
113
|
+
# will "win" and the Redis value will be ignored.
|
114
|
+
#
|
115
|
+
def get( key: )
|
116
|
+
case @get_keys_from
|
117
|
+
when :both
|
118
|
+
memcached_result = @memcached_store.get( key: key )
|
119
|
+
redis_result = @redis_store.get( key: key )
|
120
|
+
|
121
|
+
if @get_keys_from == :both && ( memcached_result.nil? || redis_result.nil? )
|
122
|
+
delete( key: key )
|
123
|
+
return nil
|
124
|
+
else
|
125
|
+
return memcached_result
|
126
|
+
end
|
127
|
+
|
128
|
+
when :memcached
|
129
|
+
return @memcached_store.get( key: key )
|
130
|
+
|
131
|
+
when :redis
|
132
|
+
return @redis_store.get( key: key )
|
133
|
+
|
134
|
+
else
|
135
|
+
raise "Hoodoo::TransientStore::Base\#get: Invalid prior value given in \#get_keys_from= of '#{ @get_keys_from.inspect }' - only ':both', ':memcached' or ':redis' are allowed"
|
136
|
+
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
# See Hoodoo::TransientStore::Base#delete for details.
|
141
|
+
#
|
142
|
+
def delete( key: )
|
143
|
+
exception = nil
|
144
|
+
|
145
|
+
begin
|
146
|
+
@memcached_store.delete( key: key )
|
147
|
+
rescue => e
|
148
|
+
exception = e
|
149
|
+
end
|
150
|
+
|
151
|
+
# But allow Redis delete to still be attempted...
|
152
|
+
|
153
|
+
begin
|
154
|
+
@redis_store.delete( key: key )
|
155
|
+
rescue => e
|
156
|
+
exception ||= e
|
157
|
+
end
|
158
|
+
|
159
|
+
if exception.nil?
|
160
|
+
return true
|
161
|
+
else
|
162
|
+
raise exception
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
# See Hoodoo::TransientStore::Base#close for details.
|
167
|
+
#
|
168
|
+
def close
|
169
|
+
@memcached_store.close() rescue nil # Rescue so that Redis "close()" is still attempted.
|
170
|
+
@redis_store.close()
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
174
|
+
|
175
|
+
Hoodoo::TransientStore.register(
|
176
|
+
as: :memcached_redis_mirror,
|
177
|
+
using: Hoodoo::TransientStore::MemcachedRedisMirror
|
178
|
+
) if ( defined?( ::Redis ) && defined?( ::Dalli ) )
|
179
|
+
|
180
|
+
end
|
181
|
+
end
|
@@ -0,0 +1,126 @@
|
|
1
|
+
########################################################################
|
2
|
+
# File:: redis.rb
|
3
|
+
# (C):: Loyalty New Zealand 2017
|
4
|
+
#
|
5
|
+
# Purpose:: Hoodoo::TransientStore plugin supporting Redis.
|
6
|
+
# ----------------------------------------------------------------------
|
7
|
+
# 01-Feb-2017 (ADH): Created.
|
8
|
+
########################################################################
|
9
|
+
|
10
|
+
begin
|
11
|
+
require 'json'
|
12
|
+
require 'redis'
|
13
|
+
rescue LoadError
|
14
|
+
end
|
15
|
+
|
16
|
+
module Hoodoo
|
17
|
+
class TransientStore
|
18
|
+
|
19
|
+
# Hoodoo::TransientStore plugin supporting {Redis}[https://redis.io]. The
|
20
|
+
# {redis-rb gem}[https://github.com/redis/redis-rb] is used for server
|
21
|
+
# communication.
|
22
|
+
#
|
23
|
+
class Redis < Hoodoo::TransientStore::Base
|
24
|
+
|
25
|
+
# See Hoodoo::TransientStore::Base::new for details.
|
26
|
+
#
|
27
|
+
# Do not instantiate this class directly. Use
|
28
|
+
# Hoodoo::TransientStore::new.
|
29
|
+
#
|
30
|
+
# The {redis-rb gem}[https://github.com/redis/redis-rb] is used to talk
|
31
|
+
# to {Redis}[https://redis.io] and requires connection UIRs with a
|
32
|
+
# +redis+ protocol, such as <tt>redis://localhost:6379</tt>.
|
33
|
+
#
|
34
|
+
# TCP keep-alive is enabled for the server connection.
|
35
|
+
#
|
36
|
+
def initialize( storage_host_uri:, namespace: )
|
37
|
+
super # Pass all arguments through -> *not* 'super()'
|
38
|
+
@client = connect_to_redis( storage_host_uri )
|
39
|
+
end
|
40
|
+
|
41
|
+
# See Hoodoo::TransientStore::Base#set for details.
|
42
|
+
#
|
43
|
+
# The payload is encoded into JSON for storage and automatically decoded
|
44
|
+
# by #get; so callers don't need to do marshalling to Strings themselves.
|
45
|
+
#
|
46
|
+
def set( key:, payload:, maximum_lifespan: )
|
47
|
+
nk = namespaced_key( key )
|
48
|
+
|
49
|
+
@client[ nk ] = JSON.fast_generate( payload )
|
50
|
+
@client.expire( nk, maximum_lifespan )
|
51
|
+
|
52
|
+
true
|
53
|
+
end
|
54
|
+
|
55
|
+
# See Hoodoo::TransientStore::Base#get for details.
|
56
|
+
#
|
57
|
+
def get( key: )
|
58
|
+
result = @client[ namespaced_key( key ) ]
|
59
|
+
return result.nil? ? nil : ( JSON.parse( result ) rescue nil )
|
60
|
+
end
|
61
|
+
|
62
|
+
# See Hoodoo::TransientStore::Base#delete for details.
|
63
|
+
#
|
64
|
+
def delete( key: )
|
65
|
+
@client.del( namespaced_key( key ) )
|
66
|
+
true
|
67
|
+
end
|
68
|
+
|
69
|
+
# See Hoodoo::TransientStore::Base#close for details.
|
70
|
+
#
|
71
|
+
def close
|
72
|
+
@client.quit()
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
|
77
|
+
# Given a simple key to Redis data (expressed as a String), return a
|
78
|
+
# namespaced version by adding the configured namespace as a prefix.
|
79
|
+
#
|
80
|
+
def namespaced_key( key )
|
81
|
+
@namespace + key
|
82
|
+
end
|
83
|
+
|
84
|
+
# Connect to Redis if possible and return the connected Redis client
|
85
|
+
# instance, else raise an exception.
|
86
|
+
#
|
87
|
+
# +host+:: Connection URI, e.g. <tt>localhost:11211</tt>.
|
88
|
+
#
|
89
|
+
def connect_to_redis( host )
|
90
|
+
exception = nil
|
91
|
+
info = nil
|
92
|
+
client = nil
|
93
|
+
|
94
|
+
begin
|
95
|
+
client = ::Redis.new(
|
96
|
+
:url => host,
|
97
|
+
:tcp_keepalive => 1
|
98
|
+
)
|
99
|
+
|
100
|
+
info = client.info( 'CPU' )
|
101
|
+
|
102
|
+
rescue => e
|
103
|
+
exception = e
|
104
|
+
|
105
|
+
end
|
106
|
+
|
107
|
+
if info.nil?
|
108
|
+
if exception.nil?
|
109
|
+
raise "Hoodoo::TransientStore::Redis: Did not get back meaningful data from Redis at '#{ host }'"
|
110
|
+
else
|
111
|
+
raise "Hoodoo::TransientStore::Redis: Cannot connect to Redis at '#{ host }': #{ exception.to_s }"
|
112
|
+
end
|
113
|
+
else
|
114
|
+
return client
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
|
120
|
+
Hoodoo::TransientStore.register(
|
121
|
+
as: :redis,
|
122
|
+
using: Hoodoo::TransientStore::Redis
|
123
|
+
) if defined?( ::Redis )
|
124
|
+
|
125
|
+
end
|
126
|
+
end
|
data/lib/hoodoo/version.rb
CHANGED
@@ -212,9 +212,7 @@ describe Hoodoo::ActiveRecord::Support do
|
|
212
212
|
expect( manual_scope ).to include( "\"r_spec_full_scope_for_test_subclasses\".\"foo\" = '#{ @test_scoping_value }'" )
|
213
213
|
end
|
214
214
|
|
215
|
-
pending 'translated'
|
216
|
-
raise "Scope verification for '\#translated'"
|
217
|
-
end
|
215
|
+
pending 'translated' # Scope verification for '#translated'
|
218
216
|
|
219
217
|
# In this last one, we actually check full_scope_for; activations are in
|
220
218
|
# the subclass so we drive the non-context versions directly to verify
|
@@ -250,9 +248,7 @@ describe Hoodoo::ActiveRecord::Support do
|
|
250
248
|
expect( manual_scope ).to include( "\"rspec_full_scope_for_test_base_with_directives_custom\".\"bar\" = '#{ @test_scoping_value }'" )
|
251
249
|
end
|
252
250
|
|
253
|
-
pending 'translated'
|
254
|
-
raise "Scope verification for '\#translated'"
|
255
|
-
end
|
251
|
+
pending 'translated' # Scope verification for '#translated'
|
256
252
|
|
257
253
|
it 'yields the same SQL' do
|
258
254
|
auto_scope = described_class.full_scope_for( RSpecFullScopeForTestBaseSubclassWithoutOverrides, @context ).to_sql()
|
@@ -281,9 +277,7 @@ describe Hoodoo::ActiveRecord::Support do
|
|
281
277
|
expect( manual_scope ).to include( "\"r_spec_full_scope_for_manually_dateds\".\"baz\" = '#{ @test_scoping_value }'" )
|
282
278
|
end
|
283
279
|
|
284
|
-
pending 'translated'
|
285
|
-
raise "Scope verification for '\#translated'"
|
286
|
-
end
|
280
|
+
pending 'translated' # Scope verification for '#translated'
|
287
281
|
|
288
282
|
it 'everything' do
|
289
283
|
auto_scope = described_class.full_scope_for( RSpecFullScopeForManuallyDated, @context ).to_sql()
|
@@ -2,10 +2,9 @@ require 'spec_helper'
|
|
2
2
|
require 'active_record'
|
3
3
|
|
4
4
|
describe Hoodoo::ActiveRecord::Translated do
|
5
|
-
pending 'is tested'
|
6
|
-
|
7
|
-
# For RCov only
|
5
|
+
pending 'is tested' # Replace with real tests!
|
8
6
|
|
7
|
+
it 'temporarily addresses RCov coverage' do
|
9
8
|
class Test
|
10
9
|
include Hoodoo::ActiveRecord::Translated
|
11
10
|
def self.all; end
|
@@ -13,7 +12,5 @@ describe Hoodoo::ActiveRecord::Translated do
|
|
13
12
|
|
14
13
|
ignored = true
|
15
14
|
Test.translated( ignored )
|
16
|
-
|
17
|
-
raise "Replace with real test"
|
18
15
|
end
|
19
16
|
end
|
@@ -9,15 +9,8 @@ describe Hoodoo::Logger::StreamWriter do
|
|
9
9
|
end
|
10
10
|
|
11
11
|
after :all do
|
12
|
-
|
13
|
-
|
14
|
-
rescue
|
15
|
-
end
|
16
|
-
|
17
|
-
begin
|
18
|
-
File.unlink( @temp_path )
|
19
|
-
rescue
|
20
|
-
end
|
12
|
+
@stream.close rescue nil
|
13
|
+
File.unlink( @temp_path ) rescue nil
|
21
14
|
end
|
22
15
|
|
23
16
|
it 'writes to streams' do
|
@@ -70,10 +70,7 @@ describe Hoodoo::Services::Middleware do
|
|
70
70
|
force_logging_to( 'test' )
|
71
71
|
Hoodoo::Services::Middleware.class_variable_set( '@@environment', @old_env )
|
72
72
|
Hoodoo::Services::Middleware.class_variable_set( '@@logger', @old_logger )
|
73
|
-
|
74
|
-
Hoodoo::Services::Middleware.remove_class_variable( '@@alchemy' )
|
75
|
-
rescue
|
76
|
-
end
|
73
|
+
Hoodoo::Services::Middleware.remove_class_variable( '@@alchemy' ) rescue nil
|
77
74
|
end
|
78
75
|
|
79
76
|
context 'custom loggers' do
|
@@ -343,14 +343,14 @@ describe Hoodoo::Services::Middleware do
|
|
343
343
|
Hoodoo::Services::Permissions::ALLOW
|
344
344
|
)
|
345
345
|
|
346
|
-
Hoodoo::
|
346
|
+
Hoodoo::TransientStore::Mocks::DalliClient.reset()
|
347
347
|
|
348
348
|
result = @session.save_to_memcached
|
349
349
|
raise "Can't save to mock Memcached (result = #{result})" unless result == :ok
|
350
350
|
end
|
351
351
|
|
352
352
|
after :each do
|
353
|
-
Hoodoo::
|
353
|
+
Hoodoo::TransientStore::Mocks::DalliClient.reset()
|
354
354
|
end
|
355
355
|
|
356
356
|
############################################################################
|
@@ -259,9 +259,9 @@ describe Hoodoo::Services::Interface do
|
|
259
259
|
it 'should complain about incorrect types' do
|
260
260
|
expect {
|
261
261
|
Hoodoo::Services::Interface::ToListDSL.new( Hoodoo::Services::Interface::ToList.new ) do
|
262
|
-
default
|
262
|
+
default( { :some => :hash } )
|
263
263
|
end
|
264
|
-
}.to raise_error(RuntimeError, "Hoodoo::Services::Interface::ToListDSL\#default requires a String or Symbol - got '
|
264
|
+
}.to raise_error(RuntimeError, "Hoodoo::Services::Interface::ToListDSL\#default requires a String or Symbol - got 'Hash'")
|
265
265
|
end
|
266
266
|
end
|
267
267
|
|