hoodoo 1.13.0 → 1.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. checksums.yaml +4 -4
  2. data/lib/hoodoo.rb +1 -0
  3. data/lib/hoodoo/services/services/session.rb +8 -104
  4. data/lib/hoodoo/transient_store.rb +19 -0
  5. data/lib/hoodoo/transient_store/mocks/dalli_client.rb +148 -0
  6. data/lib/hoodoo/transient_store/mocks/redis.rb +138 -0
  7. data/lib/hoodoo/transient_store/transient_store.rb +344 -0
  8. data/lib/hoodoo/transient_store/transient_store/base.rb +81 -0
  9. data/lib/hoodoo/transient_store/transient_store/memcached.rb +116 -0
  10. data/lib/hoodoo/transient_store/transient_store/memcached_redis_mirror.rb +181 -0
  11. data/lib/hoodoo/transient_store/transient_store/redis.rb +126 -0
  12. data/lib/hoodoo/version.rb +1 -1
  13. data/spec/active/active_record/support_spec.rb +3 -9
  14. data/spec/active/active_record/translated_spec.rb +2 -5
  15. data/spec/logger/writers/file_writer_spec.rb +1 -4
  16. data/spec/logger/writers/stream_writer_spec.rb +2 -9
  17. data/spec/services/middleware/middleware_logging_spec.rb +1 -4
  18. data/spec/services/middleware/middleware_permissions_spec.rb +2 -2
  19. data/spec/services/services/interface_spec.rb +2 -2
  20. data/spec/services/services/session_spec.rb +26 -19
  21. data/spec/transient_store/transient_store/base_spec.rb +52 -0
  22. data/spec/transient_store/transient_store/memcached_redis_mirror_spec.rb +380 -0
  23. data/spec/transient_store/transient_store/memcached_spec.rb +244 -0
  24. data/spec/transient_store/transient_store/mocks/dalli_client_spec.rb +44 -0
  25. data/spec/transient_store/transient_store/mocks/redis_spec.rb +28 -0
  26. data/spec/transient_store/transient_store/redis_spec.rb +242 -0
  27. data/spec/transient_store/transient_store_spec.rb +448 -0
  28. metadata +31 -9
@@ -0,0 +1,344 @@
1
+ ########################################################################
2
+ # File:: transient_store.rb
3
+ # (C):: Loyalty New Zealand 2017
4
+ #
5
+ # Purpose:: Provide a simple abstraction over transient storage engines
6
+ # such as Memcached or Redis, making it easier for client code
7
+ # to switch engines with very few changes.
8
+ # ----------------------------------------------------------------------
9
+ # 01-Feb-2017 (ADH): Created.
10
+ ########################################################################
11
+
12
+ module Hoodoo
13
+
14
+ # A simple abstraction over transient storage engines such as
15
+ # {Memcached}[https://memcached.org] or {Redis}[https://redis.io], making it
16
+ # it easier for client code to switch engines with very few changes. If the
17
+ # storage engine chosen when creating instances of this object is defined in
18
+ # application-wide configuration data, all you would need to do is change the
19
+ # configuration for all new TransientStore instances to use the new engine.
20
+ #
21
+ class TransientStore
22
+
23
+ ###########################################################################
24
+ # Class methods
25
+ ###########################################################################
26
+
27
+ # Register a new storage engine plugin class. It _MUST_ inherit from and
28
+ # thus follow the template laid out in Hoodoo::TransientStore::Base.
29
+ #
30
+ # _Named_ parameters are:
31
+ #
32
+ # +as+:: The name, as a Symbol, for the ::new +storage_engine+
33
+ # input parameter, to select this plugin.
34
+ #
35
+ # +using+:: The class reference of the Hoodoo::TransientStore::Base
36
+ # subclass to be associated with the name given in +as+.
37
+ #
38
+ # Example:
39
+ #
40
+ # Hoodoo::TransientStore.register(
41
+ # as: :memcached,
42
+ # using: Hoodoo::TransientStore::Memcached
43
+ # )
44
+ #
45
+ def self.register( as:, using: )
46
+ as = as.to_sym
47
+
48
+ @@supported_storage_engines = {} unless defined?( @@supported_storage_engines )
49
+
50
+ unless using < Hoodoo::TransientStore::Base
51
+ raise "Hoodoo::TransientStore.register requires a Hoodoo::TransientStore::Base subclass - got '#{ using.to_s }'"
52
+ end
53
+
54
+ if @@supported_storage_engines.has_key?( as )
55
+ raise "Hoodoo::TransientStore.register: A storage engine called '#{ as }' is already registered"
56
+ end
57
+
58
+ @@supported_storage_engines[ as ] = using
59
+ end
60
+
61
+ # Remove a storage engine plugin class from the supported collection. Any
62
+ # existing Hoodoo::TransientStore instances using the removed class will
63
+ # not be affected, but new instances cannot be made.
64
+ #
65
+ # _Named_ parameters are:
66
+ #
67
+ # +as+:: The value given to #register in the corresponding +as+ parameter.
68
+ #
69
+ def self.deregister( as: )
70
+ @@supported_storage_engines.delete( as )
71
+ end
72
+
73
+ # Return an array of the names of all supported storage engine names known
74
+ # to the Hoodoo::TransientStore class. Any one of those names can be used
75
+ # with the ::new +storage_engine+ parameter.
76
+ #
77
+ def self.supported_storage_engines
78
+ @@supported_storage_engines.keys()
79
+ end
80
+
81
+ ###########################################################################
82
+ # Instance methods
83
+ ###########################################################################
84
+
85
+ # Read this instance's storage engine; see ::supported_storage_engines and
86
+ # ::new.
87
+ #
88
+ attr_reader :storage_engine
89
+
90
+ # Read the storage engine insteance for the #storage_engine - this allows
91
+ # engine-specific configuration to be set where available, though this is
92
+ # strongly discouraged as it couples client code to the engine in use,
93
+ # defeating the main rationale behind the TransientStore abstraction.
94
+ #
95
+ attr_reader :storage_engine_instance
96
+
97
+ # Read this instance's default item maximum lifespan, in seconds. See
98
+ # also ::new.
99
+ #
100
+ attr_reader :default_maximum_lifespan
101
+
102
+ # Read this instance's default storage namespace, as a String. See also
103
+ # ::new.
104
+ #
105
+ attr_reader :default_namespace
106
+
107
+ # Instantiate a new Transient storage object through which temporary data
108
+ # can be stored or retrieved.
109
+ #
110
+ # The TransientStore abstraction is a high level and simple abstraction over
111
+ # heterogenous data storage engines. It does not expose the many subtle
112
+ # configuration settings usually available in such. If you need to take
113
+ # advantage of those at an item storage level, you'll need to use a lower
114
+ # level interface and thus lock your code to the engine of choice.
115
+ #
116
+ # Engine plug-ins are recommended to attempt to gain and test a connection
117
+ # to the storage engine when this object is constructed, so if building a
118
+ # TransientStore instance, ensure your chosen storage engine is running
119
+ # first. Exceptions may be raised by storage engines, so you will probably
120
+ # want to catch those with more civilised error handling code.
121
+ #
122
+ # _Named_ parameters are:
123
+ #
124
+ # +storage_engine+:: An entry from ::supported_storage_engines.
125
+ #
126
+ # +storage_host_uri+:: The engine-dependent connection URI. Consult
127
+ # documentation for your chosen engine to find
128
+ # out its connection URI requirements, along
129
+ # with the documentation for the constructor
130
+ # method of the plug-in in use, since in some
131
+ # cases requirements may be unusual (e.g. in
132
+ # Hoodoo::TransientStore::MemcachedRedisMirror).
133
+ #
134
+ # +default_maximum_lifespan+:: The default time-to-live for data items, in,
135
+ # seconds; can be overridden per item; default
136
+ # is 604800 seconds or 7 days.
137
+ #
138
+ # +default_namespace+:: Storage engine keys are namespaced with
139
+ # +nz_co_loyalty_hoodoo_transient_store_+ by
140
+ # default, though this can be overridden here.
141
+ # Pass a String or Symbol.
142
+ #
143
+ def initialize(
144
+ storage_engine:,
145
+ storage_host_uri:,
146
+ default_maximum_lifespan: 604800,
147
+ default_namespace: 'nz_co_loyalty_hoodoo_transient_store_'
148
+ )
149
+
150
+ unless self.class.supported_storage_engines().include?( storage_engine )
151
+
152
+ # Be kind and use 'inspect' to indicate that we expect Symbols here
153
+ # in the exception, because of the arising leading ':' in the output.
154
+ #
155
+ engines = self.class.supported_storage_engines().map { | symbol | "'#{ symbol.inspect }'" }
156
+ allowed = engines.join( ', ' )
157
+
158
+ raise "Hoodoo::TransientStore: Unrecognised storage engine '#{ storage_engine.inspect }' requested; allowed values: #{ allowed }"
159
+ end
160
+
161
+ @default_maximum_lifespan = default_maximum_lifespan
162
+ @default_namespace = ( default_namespace || '' ).to_s()
163
+ @storage_engine = storage_engine
164
+ @storage_engine_instance = @@supported_storage_engines[ storage_engine ].new(
165
+ storage_host_uri: storage_host_uri,
166
+ namespace: @default_namespace
167
+ )
168
+
169
+ end
170
+
171
+ # Set (write) a given payload into the storage engine with the given
172
+ # payload and maximum lifespan.
173
+ #
174
+ # Payloads must only contain simple types such as Hash, Array, String and
175
+ # Integer. Complex types like Symbol, Date, Float, BigDecimal or custom
176
+ # objects are unlikely to serialise properly but since this depends upon
177
+ # the storage engine in use, errors may or may not be raised for misuse.
178
+ #
179
+ # Storage engines usually have a maximum payload size limit; consult your
180
+ # engine administrator for information. For example, the default - but
181
+ # reconfigurable - maximum payload size for Memcached is 1MB.
182
+ #
183
+ # For maximum possible compatibility:
184
+ #
185
+ # * Use only Hash payloads with String key/value paids and no nesting. You
186
+ # may choose to marshal the data into a String manually for unusual data
187
+ # requirements, manually converting back when reading stored data.
188
+ #
189
+ # * Keep the payload size as small as possible - large objects belong in
190
+ # bulk storage engines such as Amazon S3.
191
+ #
192
+ # These are only guidelines though - heterogenous storage engine support
193
+ # and the ability of system administrators to arbitrarily configure those
194
+ # storage engines makes it impossible to be more precise.
195
+ #
196
+ # Returns:
197
+ #
198
+ # * +true+ if storage was successful
199
+ # * +false+ if storage failed but the reason is unknown
200
+ # * An +Exception+ instance if storage failed and the storage engine
201
+ # raised an exception describing the problem.
202
+ #
203
+ # _Named_ parameters are:
204
+ #
205
+ # +key+:: Storage key to use in the engine, which is then used
206
+ # in subsequent calls to #get and possibly eventually
207
+ # to #delete. Only non-empty Strings or Symbols are
208
+ # permitted, else an exception will be raised.
209
+ #
210
+ # +payload+:: Payload data to store under the given +key+. A flat
211
+ # Hash is recommended rather than simple types such as
212
+ # String (unless marshalling a complex type into such)
213
+ # in order to make potential additions to stored data
214
+ # easier to implement. Note that +nil+ is prohibited.
215
+ #
216
+ # +maximum_lifespan+:: Optional maximum lifespan, seconds. Storage engines
217
+ # may chooset to evict payloads sooner than this; it
218
+ # is a maximum time, not a guarantee. Omit to use this
219
+ # TransientStore instance's default value - see ::new.
220
+ # If you know you no longer need a piece of data at a
221
+ # particular point in the execution flow of your code,
222
+ # explicitly delete it via #delete rather than leaving
223
+ # it to expire. This maximises the storage engine's
224
+ # pool free space and so minimises the chance of early
225
+ # item eviction.
226
+ #
227
+ def set( key:, payload:, maximum_lifespan: nil )
228
+ key = normalise_key( key, 'set' )
229
+
230
+ if payload.nil?
231
+ raise "Hoodoo::TransientStore\#set: Payloads of 'nil' are prohibited"
232
+ end
233
+
234
+ maximum_lifespan ||= @default_maximum_lifespan
235
+
236
+ begin
237
+ result = @storage_engine_instance.set(
238
+ key: key,
239
+ payload: payload,
240
+ maximum_lifespan: maximum_lifespan
241
+ )
242
+
243
+ if result != true && result != false
244
+ raise "Hoodoo::TransientStore\#set: Engine '#{ @storage_engine }' returned an invalid response"
245
+ end
246
+
247
+ rescue => e
248
+ result = e
249
+
250
+ end
251
+
252
+ return result
253
+ end
254
+
255
+ # Retrieve data previously stored with #set.
256
+ #
257
+ # _Named_ parameters are:
258
+ #
259
+ # +key+:: Key previously given to #set.
260
+ #
261
+ # Returns +nil+ if the item is not found - either the key is wrong, the
262
+ # stored data has expired or the stored data has been evicted early from
263
+ # the storage engine's pool.
264
+ #
265
+ # Only non-empty String or Symbol keys are permitted, else an exception
266
+ # will be raised.
267
+ #
268
+ def get( key: )
269
+ key = normalise_key( key, 'get' )
270
+ @storage_engine_instance.get( key: key ) rescue nil
271
+ end
272
+
273
+ # Delete data previously stored with #set.
274
+ #
275
+ # _Named_ parameters are:
276
+ #
277
+ # +key+:: Key previously given to #set.
278
+ #
279
+ # Returns:
280
+ #
281
+ # * +true+ if deletion was successful, if the item has already expired or
282
+ # if the key is simply not recognised so there is no more work to do.
283
+ # * +false+ if deletion failed but the reason is unknown.
284
+ # * An +Exception+ instance if deletion failed and the storage engine
285
+ # raised an exception describing the problem.
286
+ #
287
+ # Only non-empty String or Symbol keys are permitted, else an exception
288
+ # will be raised.
289
+ #
290
+ def delete( key: )
291
+ key = normalise_key( key, 'delete' )
292
+
293
+ begin
294
+ result = @storage_engine_instance.delete( key: key )
295
+
296
+ if result != true && result != false
297
+ raise "Hoodoo::TransientStore\#delete: Engine '#{ @storage_engine }' returned an invalid response"
298
+ end
299
+
300
+ rescue => e
301
+ result = e
302
+
303
+ end
304
+
305
+ return result
306
+ end
307
+
308
+ # If you aren't going to use this instance again, it is good manners to
309
+ # immediately close its connection(s) to any storage engines by calling
310
+ # here.
311
+ #
312
+ # No useful return value is generated and exceptions are ignored.
313
+ #
314
+ def close
315
+ @storage_engine_instance.close() rescue nil
316
+ end
317
+
318
+ private
319
+
320
+ # Given a storage key, make sure it's a String or Symbol, coerce to a
321
+ # String and ensure it isn't empty. Returns the non-empty String version.
322
+ # Raises exceptions for bad input classes or empty keys.
323
+ #
324
+ # +key+:: Key to normalise.
325
+ #
326
+ # +calling_method_name+:: Name of calling method to declare in exception
327
+ # messages, to aid callers in debugging.
328
+ #
329
+ def normalise_key( key, calling_method_name )
330
+ unless key.is_a?( String ) || key.is_a?( Symbol )
331
+ raise "Hoodoo::TransientStore\##{ calling_method_name }: Keys must be of String or Symbol class; you provided '#{ key.class }'"
332
+ end
333
+
334
+ key = key.to_s
335
+
336
+ if key.empty?
337
+ raise "Hoodoo::TransientStore\##{ calling_method_name }: Empty String or Symbol keys are prohibited"
338
+ end
339
+
340
+ return key
341
+ end
342
+
343
+ end
344
+ end
@@ -0,0 +1,81 @@
1
+ ########################################################################
2
+ # File:: base.rb
3
+ # (C):: Loyalty New Zealand 2017
4
+ #
5
+ # Purpose:: Base class for Base class for Hoodoo::TransientStore plugins.
6
+ # ----------------------------------------------------------------------
7
+ # 01-Feb-2017 (ADH): Created.
8
+ ########################################################################
9
+
10
+ module Hoodoo
11
+ class TransientStore
12
+
13
+ # Base class for Hoodoo::TransientStore plugins. This is in effect just a
14
+ # template / abstract class, providing a source-level guideline for plug-in
15
+ # authors. See also out-of-the-box existing plug-ins as worked examples.
16
+ #
17
+ class Base
18
+
19
+ # Base class template for a constructor. Subclasses should try to
20
+ # establish a connection with their storage engine(s) here and raise
21
+ # exceptions if things go wrong.
22
+ #
23
+ # +storage_host_uri+:: The engine-dependent connection URI. See
24
+ # Hoodoo::TransientStore::new for details.
25
+ #
26
+ # +namespace+:: The storage key namespace to use, as a String.
27
+ # See Hoodoo::TransientStore::new for details.
28
+ #
29
+ def initialize( storage_host_uri:, namespace: )
30
+ @storage_host_uri = storage_host_uri
31
+ @namespace = namespace
32
+ end
33
+
34
+ # Base class template for the plug-in's back-end implementation of
35
+ # Hoodoo::TransientStore#set - see that for details.
36
+ #
37
+ # The implementation is free to raise an exception if an error is
38
+ # encountered while trying to set the data - this will be caught and
39
+ # returned by Hoodoo::TransientStore#set. Otherwise return +true+ on
40
+ # success or +false+ for failures of unknown origin.
41
+ #
42
+ def set( key:, payload:, maximum_lifespan: )
43
+ raise 'Subclasses must implement Hoodoo::TransientStore::Base#set'
44
+ end
45
+
46
+ # Base class template for the plug-in's back-end implementation of
47
+ # Hoodoo::TransientStore#get - see that for details. Returns +nil+ if
48
+ # no data is found for the given key, or if data is explicitly +nil+.
49
+ #
50
+ # The implementation is free to raise an exception if an error is
51
+ # encountered while trying to get the data - this will be caught and
52
+ # +nil+ returned by Hoodoo::TransientStore#get.
53
+ #
54
+ def get( key: )
55
+ raise 'Subclasses must implement Hoodoo::TransientStore::Base#get'
56
+ end
57
+
58
+ # Base class template for the plug-in's back-end implementation of
59
+ # Hoodoo::TransientStore#delete - see that for details.
60
+ #
61
+ # The implementation is free to raise an exception if an error is
62
+ # encountered while trying to get the data - this will be caught and
63
+ # ignored by Hoodoo::TransientStore#delete. Otherwise return +true+ on
64
+ # success or +false+ for failures of unknown origin.
65
+ #
66
+ def delete( key: )
67
+ raise 'Subclasses must implement Hoodoo::TransientStore::Base#delete'
68
+ end
69
+
70
+ # Base class template for the plug-in's back-end implementation of
71
+ # Hoodoo::TransientStore#close - see that for details.
72
+ #
73
+ # Any exception raised will be ignored by Hoodoo::TransientStore#close.
74
+ #
75
+ def close
76
+ raise 'Subclasses must implement Hoodoo::TransientStore::Base#close'
77
+ end
78
+
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,116 @@
1
+ ########################################################################
2
+ # File:: memcached.rb
3
+ # (C):: Loyalty New Zealand 2017
4
+ #
5
+ # Purpose:: Hoodoo::TransientStore plugin supporting Memcached.
6
+ # ----------------------------------------------------------------------
7
+ # 01-Feb-2017 (ADH): Created.
8
+ ########################################################################
9
+
10
+ begin
11
+ require 'dalli'
12
+ rescue LoadError
13
+ end
14
+
15
+ module Hoodoo
16
+ class TransientStore
17
+
18
+ # Hoodoo::TransientStore plugin for {Memcached}[https://memcached.org]. The
19
+ # {Dalli gem}[https://github.com/petergoldstein/dalli] is used for server
20
+ # communication.
21
+ #
22
+ class Memcached < Hoodoo::TransientStore::Base
23
+
24
+ # See Hoodoo::TransientStore::Base::new for details.
25
+ #
26
+ # Do not instantiate this class directly. Use
27
+ # Hoodoo::TransientStore::new.
28
+ #
29
+ # The {Dalli gem}[https://github.com/petergoldstein/dalli] is used to
30
+ # talk to {Memcached}[https://memcached.org] and accepts connection UIRs
31
+ # of simple, terse forms such as <tt>'localhost:11211'</tt>. Connections
32
+ # are configured with JSON serialisation, compression off and a forced
33
+ # namespace of +nz_co_loyalty_hoodoo_transient_store_+ to avoid collision
34
+ # of data stored with this object and other data that may be in the
35
+ # Memcached instance identified by +storage_host_uri+.
36
+ #
37
+ def initialize( storage_host_uri:, namespace: )
38
+ super # Pass all arguments through -> *not* 'super()'
39
+ @client = connect_to_memcached( storage_host_uri, namespace )
40
+ end
41
+
42
+ # See Hoodoo::TransientStore::Base#set for details.
43
+ #
44
+ def set( key:, payload:, maximum_lifespan: )
45
+ @client.set( key, payload, maximum_lifespan )
46
+ true
47
+ end
48
+
49
+ # See Hoodoo::TransientStore::Base#get for details.
50
+ #
51
+ def get( key: )
52
+ @client.get( key )
53
+ end
54
+
55
+ # See Hoodoo::TransientStore::Base#delete for details.
56
+ #
57
+ def delete( key: )
58
+ @client.delete( key )
59
+ true
60
+ end
61
+
62
+ # See Hoodoo::TransientStore::Base#close for details.
63
+ #
64
+ def close
65
+ @client.close()
66
+ end
67
+
68
+ private
69
+
70
+ # Connect to Memcached if possible and return the connected Dalli client
71
+ # instance, else raise an exception.
72
+ #
73
+ # +host+:: Connection URI, e.g. <tt>localhost:11211</tt>.
74
+ #
75
+ def connect_to_memcached( host, namespace )
76
+ exception = nil
77
+ stats = nil
78
+ client = nil
79
+
80
+ begin
81
+ client = ::Dalli::Client.new(
82
+ host,
83
+ {
84
+ :compress => false,
85
+ :serializer => JSON,
86
+ :namespace => namespace
87
+ }
88
+ )
89
+
90
+ stats = client.stats()
91
+
92
+ rescue => e
93
+ exception = e
94
+
95
+ end
96
+
97
+ if stats.nil?
98
+ if exception.nil?
99
+ raise "Hoodoo::TransientStore::Memcached: Did not get back meaningful data from Memcached at '#{ host }'"
100
+ else
101
+ raise "Hoodoo::TransientStore::Memcached: Cannot connect to Memcached at '#{ host }': #{ exception.to_s }"
102
+ end
103
+ else
104
+ return client
105
+ end
106
+ end
107
+
108
+ end
109
+
110
+ Hoodoo::TransientStore.register(
111
+ as: :memcached,
112
+ using: Hoodoo::TransientStore::Memcached
113
+ ) if defined?( ::Dalli )
114
+
115
+ end
116
+ end