faulty 0.1.4 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +49 -0
- data/.rubocop.yml +9 -0
- data/CHANGELOG.md +55 -0
- data/Gemfile +8 -3
- data/README.md +883 -310
- data/bin/check-version +5 -1
- data/faulty.gemspec +1 -1
- data/lib/faulty.rb +167 -43
- data/lib/faulty/cache.rb +3 -1
- data/lib/faulty/cache/auto_wire.rb +58 -0
- data/lib/faulty/cache/circuit_proxy.rb +61 -0
- data/lib/faulty/cache/default.rb +10 -21
- data/lib/faulty/cache/fault_tolerant_proxy.rb +15 -4
- data/lib/faulty/cache/interface.rb +1 -1
- data/lib/faulty/cache/mock.rb +1 -1
- data/lib/faulty/cache/null.rb +1 -1
- data/lib/faulty/cache/rails.rb +9 -10
- data/lib/faulty/circuit.rb +31 -16
- data/lib/faulty/error.rb +29 -7
- data/lib/faulty/events.rb +1 -1
- data/lib/faulty/events/callback_listener.rb +1 -1
- data/lib/faulty/events/honeybadger_listener.rb +1 -1
- data/lib/faulty/events/listener_interface.rb +1 -1
- data/lib/faulty/events/log_listener.rb +5 -6
- data/lib/faulty/events/notifier.rb +1 -1
- data/lib/faulty/immutable_options.rb +1 -1
- data/lib/faulty/patch.rb +154 -0
- data/lib/faulty/patch/base.rb +46 -0
- data/lib/faulty/patch/redis.rb +60 -0
- data/lib/faulty/result.rb +2 -2
- data/lib/faulty/status.rb +3 -2
- data/lib/faulty/storage.rb +4 -1
- data/lib/faulty/storage/auto_wire.rb +107 -0
- data/lib/faulty/storage/circuit_proxy.rb +64 -0
- data/lib/faulty/storage/fallback_chain.rb +207 -0
- data/lib/faulty/storage/fault_tolerant_proxy.rb +52 -57
- data/lib/faulty/storage/interface.rb +3 -2
- data/lib/faulty/storage/memory.rb +8 -4
- data/lib/faulty/storage/redis.rb +75 -13
- data/lib/faulty/version.rb +2 -2
- metadata +14 -7
- data/.travis.yml +0 -46
- data/lib/faulty/scope.rb +0 -117
@@ -1,14 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Cache
|
5
5
|
# A wrapper for cache backends that may raise errors
|
6
6
|
#
|
7
|
-
# {
|
7
|
+
# {Faulty#initialize} automatically wraps all non-fault-tolerant cache backends with
|
8
8
|
# this class.
|
9
9
|
#
|
10
10
|
# If the cache backend raises a `StandardError`, it will be captured and
|
11
|
-
# sent to the notifier.
|
11
|
+
# sent to the notifier. Reads errors will return `nil`, and writes will be
|
12
|
+
# a no-op.
|
12
13
|
class FaultTolerantProxy
|
13
14
|
attr_reader :options
|
14
15
|
|
@@ -36,6 +37,16 @@ module Faulty
|
|
36
37
|
@options = Options.new(options, &block)
|
37
38
|
end
|
38
39
|
|
40
|
+
# Wrap a cache in a FaultTolerantProxy unless it's already fault tolerant
|
41
|
+
#
|
42
|
+
# @param cache [Cache::Interface] The cache to maybe wrap
|
43
|
+
# @return [Cache::Interface] The original cache or a {FaultTolerantProxy}
|
44
|
+
def self.wrap(cache, **options, &block)
|
45
|
+
return cache if cache.fault_tolerant?
|
46
|
+
|
47
|
+
new(cache, **options, &block)
|
48
|
+
end
|
49
|
+
|
39
50
|
# Read from the cache safely
|
40
51
|
#
|
41
52
|
# If the backend raises a `StandardError`, this will return `nil`.
|
@@ -58,7 +69,7 @@ module Faulty
|
|
58
69
|
# @return [void]
|
59
70
|
def write(key, value, expires_in: nil)
|
60
71
|
@cache.write(key, value, expires_in: expires_in)
|
61
|
-
rescue StandardError
|
72
|
+
rescue StandardError => e
|
62
73
|
options.notifier.notify(:cache_failure, key: key, action: :write, error: e)
|
63
74
|
nil
|
64
75
|
end
|
data/lib/faulty/cache/mock.rb
CHANGED
data/lib/faulty/cache/null.rb
CHANGED
data/lib/faulty/cache/rails.rb
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Cache
|
5
5
|
# A wrapper for a Rails or ActiveSupport cache
|
6
6
|
#
|
7
7
|
class Rails
|
8
|
+
extend Forwardable
|
9
|
+
|
8
10
|
# @param cache The Rails cache to wrap
|
9
11
|
# @param fault_tolerant [Boolean] Whether the Rails cache is
|
10
12
|
# fault_tolerant. See {#fault_tolerant?} for more details
|
@@ -13,15 +15,12 @@ module Faulty
|
|
13
15
|
@fault_tolerant = fault_tolerant
|
14
16
|
end
|
15
17
|
|
16
|
-
#
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
def write(key, value, expires_in: nil)
|
23
|
-
@cache.write(key, value, expires_in: expires_in)
|
24
|
-
end
|
18
|
+
# @!method read(key)
|
19
|
+
# (see Faulty::Cache::Interface#read)
|
20
|
+
#
|
21
|
+
# @!method write(key, value, expires_in: expires_in)
|
22
|
+
# (see Faulty::Cache::Interface#write)
|
23
|
+
def_delegators :@cache, :read, :write
|
25
24
|
|
26
25
|
# Although ActiveSupport cache implementations are fault-tolerant,
|
27
26
|
# Rails.cache is not guranteed to be fault tolerant. For this reason,
|
data/lib/faulty/circuit.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
# Runs code protected by a circuit breaker
|
5
5
|
#
|
6
6
|
# https://www.martinfowler.com/bliki/CircuitBreaker.html
|
@@ -50,6 +50,9 @@ module Faulty
|
|
50
50
|
# @!attribute [r] cool_down
|
51
51
|
# @return [Integer] The number of seconds the circuit will
|
52
52
|
# stay open after it is tripped. Default 300.
|
53
|
+
# @!attribute [r] error_module
|
54
|
+
# @return [Module] Used by patches to set the namespace module for
|
55
|
+
# the faulty errors that will be raised. Default `Faulty`
|
53
56
|
# @!attribute [r] evaluation_window
|
54
57
|
# @return [Integer] The number of seconds of history that
|
55
58
|
# will be evaluated to determine the failure rate for a circuit.
|
@@ -66,14 +69,19 @@ module Faulty
|
|
66
69
|
# @return [Error, Array<Error>] An array of errors that are considered circuit
|
67
70
|
# failures. Default `[StandardError]`.
|
68
71
|
# @!attribute [r] exclude
|
69
|
-
# @return [Error, Array<Error>] An array of errors that will be
|
70
|
-
#
|
72
|
+
# @return [Error, Array<Error>] An array of errors that will not be
|
73
|
+
# captured by Faulty. These errors will not be considered circuit
|
74
|
+
# failures. Default `[]`.
|
71
75
|
# @!attribute [r] cache
|
72
|
-
# @return [Cache::Interface] The cache backend. Default
|
76
|
+
# @return [Cache::Interface] The cache backend. Default
|
77
|
+
# `Cache::Null.new`. Unlike {Faulty#initialize}, this is not wrapped in
|
78
|
+
# {Cache::AutoWire} by default.
|
73
79
|
# @!attribute [r] notifier
|
74
80
|
# @return [Events::Notifier] A Faulty notifier. Default `Events::Notifier.new`
|
75
81
|
# @!attribute [r] storage
|
76
|
-
# @return [Storage::Interface] The storage backend. Default
|
82
|
+
# @return [Storage::Interface] The storage backend. Default
|
83
|
+
# `Storage::Memory.new`. Unlike {Faulty#initialize}, this is not wrapped
|
84
|
+
# in {Storage::AutoWire} by default.
|
77
85
|
Options = Struct.new(
|
78
86
|
:cache_expires_in,
|
79
87
|
:cache_refreshes_after,
|
@@ -83,6 +91,7 @@ module Faulty
|
|
83
91
|
:rate_threshold,
|
84
92
|
:sample_threshold,
|
85
93
|
:errors,
|
94
|
+
:error_module,
|
86
95
|
:exclude,
|
87
96
|
:cache,
|
88
97
|
:notifier,
|
@@ -98,6 +107,7 @@ module Faulty
|
|
98
107
|
cache_refreshes_after: 900,
|
99
108
|
cool_down: 300,
|
100
109
|
errors: [StandardError],
|
110
|
+
error_module: Faulty,
|
101
111
|
exclude: [],
|
102
112
|
evaluation_window: 60,
|
103
113
|
rate_threshold: 0.5,
|
@@ -110,6 +120,7 @@ module Faulty
|
|
110
120
|
cache
|
111
121
|
cool_down
|
112
122
|
errors
|
123
|
+
error_module
|
113
124
|
exclude
|
114
125
|
evaluation_window
|
115
126
|
rate_threshold
|
@@ -208,9 +219,11 @@ module Faulty
|
|
208
219
|
cached_value = cache_read(cache)
|
209
220
|
# return cached unless cached.nil?
|
210
221
|
return cached_value if !cached_value.nil? && !cache_should_refresh?(cache)
|
211
|
-
return run_skipped(cached_value) unless status.can_run?
|
212
222
|
|
213
|
-
|
223
|
+
current_status = status
|
224
|
+
return run_skipped(cached_value) unless current_status.can_run?
|
225
|
+
|
226
|
+
run_exec(current_status, cached_value, cache, &block)
|
214
227
|
end
|
215
228
|
|
216
229
|
# Force the circuit to stay open until unlocked
|
@@ -277,7 +290,7 @@ module Faulty
|
|
277
290
|
# @return The result from cache if available
|
278
291
|
def run_skipped(cached_value)
|
279
292
|
skipped!
|
280
|
-
raise OpenCircuitError.new(nil, self) if cached_value.nil?
|
293
|
+
raise options.error_module::OpenCircuitError.new(nil, self) if cached_value.nil?
|
281
294
|
|
282
295
|
cached_value
|
283
296
|
end
|
@@ -287,26 +300,27 @@ module Faulty
|
|
287
300
|
# @param cached_value The cached value if one is available
|
288
301
|
# @param cache_key [String, nil] The cache key if one is given
|
289
302
|
# @return The run result
|
290
|
-
def run_exec(cached_value, cache_key)
|
303
|
+
def run_exec(status, cached_value, cache_key)
|
291
304
|
result = yield
|
292
|
-
success!
|
305
|
+
success!(status)
|
293
306
|
cache_write(cache_key, result)
|
294
307
|
result
|
295
308
|
rescue *options.errors => e
|
296
309
|
raise if options.exclude.any? { |ex| e.is_a?(ex) }
|
297
310
|
|
298
311
|
if cached_value.nil?
|
299
|
-
raise CircuitTrippedError.new(nil, self) if failure!(e)
|
312
|
+
raise options.error_module::CircuitTrippedError.new(nil, self) if failure!(status, e)
|
300
313
|
|
301
|
-
raise CircuitFailureError.new(nil, self)
|
314
|
+
raise options.error_module::CircuitFailureError.new(nil, self)
|
302
315
|
else
|
303
316
|
cached_value
|
304
317
|
end
|
305
318
|
end
|
306
319
|
|
307
320
|
# @return [Boolean] True if the circuit transitioned to closed
|
308
|
-
def success!
|
309
|
-
|
321
|
+
def success!(status)
|
322
|
+
entries = storage.entry(self, Faulty.current_time, true)
|
323
|
+
status = Status.from_entries(entries, **status.to_h)
|
310
324
|
closed = false
|
311
325
|
closed = close! if should_close?(status)
|
312
326
|
|
@@ -315,8 +329,9 @@ module Faulty
|
|
315
329
|
end
|
316
330
|
|
317
331
|
# @return [Boolean] True if the circuit transitioned to open
|
318
|
-
def failure!(error)
|
319
|
-
|
332
|
+
def failure!(status, error)
|
333
|
+
entries = storage.entry(self, Faulty.current_time, false)
|
334
|
+
status = Status.from_entries(entries, **status.to_h)
|
320
335
|
options.notifier.notify(:circuit_failure, circuit: self, status: status, error: error)
|
321
336
|
|
322
337
|
opened = if status.half_open?
|
data/lib/faulty/error.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
# The base error for all Faulty errors
|
5
5
|
class FaultyError < StandardError; end
|
6
6
|
|
@@ -20,19 +20,21 @@ module Faulty
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
# Raised if getting the default
|
24
|
-
class
|
23
|
+
# Raised if getting the default instance without initializing one
|
24
|
+
class MissingDefaultInstanceError < FaultyError
|
25
25
|
def initialize(message = nil)
|
26
|
-
message ||= 'No default
|
26
|
+
message ||= 'No default instance. Create one with init or get your instance with Faulty[:name]'
|
27
27
|
super(message)
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
|
-
#
|
32
|
-
#
|
33
|
-
|
31
|
+
# Included in faulty circuit errors to provide common features for
|
32
|
+
# native and patched errors
|
33
|
+
module CircuitErrorBase
|
34
34
|
attr_reader :circuit
|
35
35
|
|
36
|
+
# @param message [String]
|
37
|
+
# @param circuit [Circuit] The circuit that raised the error
|
36
38
|
def initialize(message, circuit)
|
37
39
|
message ||= %(circuit error for "#{circuit.name}")
|
38
40
|
@circuit = circuit
|
@@ -41,6 +43,12 @@ module Faulty
|
|
41
43
|
end
|
42
44
|
end
|
43
45
|
|
46
|
+
# The base error for all errors raised during circuit runs
|
47
|
+
#
|
48
|
+
class CircuitError < FaultyError
|
49
|
+
include CircuitErrorBase
|
50
|
+
end
|
51
|
+
|
44
52
|
# Raised when running a circuit that is already open
|
45
53
|
class OpenCircuitError < CircuitError; end
|
46
54
|
|
@@ -59,8 +67,22 @@ module Faulty
|
|
59
67
|
# Raised if calling get or error on a result without checking it
|
60
68
|
class UncheckedResultError < FaultyError; end
|
61
69
|
|
70
|
+
# An error that wraps multiple other errors
|
71
|
+
class FaultyMultiError < FaultyError
|
72
|
+
def initialize(message, errors)
|
73
|
+
message = "#{message}: #{errors.map(&:message).join(', ')}"
|
74
|
+
super(message)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
62
78
|
# Raised if getting the wrong result type.
|
63
79
|
#
|
64
80
|
# For example, calling get on an error result will raise this
|
65
81
|
class WrongResultError < FaultyError; end
|
82
|
+
|
83
|
+
# Raised if a FallbackChain partially fails
|
84
|
+
class PartialFailureError < FaultyMultiError; end
|
85
|
+
|
86
|
+
# Raised if all FallbackChain backends fail
|
87
|
+
class AllFailedError < FaultyMultiError; end
|
66
88
|
end
|
data/lib/faulty/events.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
3
|
+
class Faulty
|
4
4
|
module Events
|
5
5
|
# A default listener that logs Faulty events
|
6
6
|
class LogListener
|
@@ -72,11 +72,10 @@ module Faulty
|
|
72
72
|
end
|
73
73
|
|
74
74
|
def storage_failure(payload)
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
)
|
75
|
+
extra = {}
|
76
|
+
extra[:circuit] = payload[:circuit].name if payload.key?(:circuit)
|
77
|
+
extra[:error] = payload[:error].message
|
78
|
+
log(:error, 'Storage failure', payload[:action], extra)
|
80
79
|
end
|
81
80
|
|
82
81
|
def log(level, msg, action, extra = {})
|
data/lib/faulty/patch.rb
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'faulty/patch/base'
|
4
|
+
|
5
|
+
class Faulty
|
6
|
+
# Automatic wrappers for common core dependencies like database connections
|
7
|
+
# or caches
|
8
|
+
module Patch
|
9
|
+
class << self
|
10
|
+
# Create a circuit from a configuration hash
|
11
|
+
#
|
12
|
+
# This is intended to be used in contexts where the user passes in
|
13
|
+
# something like a connection hash to a third-party library. For example
|
14
|
+
# the Redis patch hooks into the normal Redis connection options to add
|
15
|
+
# a `:faulty` key, which is a hash of faulty circuit options. This is
|
16
|
+
# slightly different from the normal Faulty circuit options because
|
17
|
+
# we also accept an `:instance` key which is a faulty instance.
|
18
|
+
#
|
19
|
+
# @example
|
20
|
+
# # We pass in a faulty instance along with some circuit options
|
21
|
+
# Patch.circuit_from_hash(
|
22
|
+
# :mysql,
|
23
|
+
# { host: 'localhost', faulty: {
|
24
|
+
# name: 'my_mysql', # A custom circuit name can be included
|
25
|
+
# instance: Faulty.new,
|
26
|
+
# sample_threshold: 5
|
27
|
+
# }
|
28
|
+
# }
|
29
|
+
# )
|
30
|
+
#
|
31
|
+
# @example
|
32
|
+
# # instance can be a registered faulty instance referenced by a string
|
33
|
+
# or symbol
|
34
|
+
# Faulty.register(:db_faulty, Faulty.new)
|
35
|
+
# Patch.circuit_from_hash(
|
36
|
+
# :mysql,
|
37
|
+
# { host: 'localhost', faulty: { instance: :db_faulty } }
|
38
|
+
# )
|
39
|
+
# @example
|
40
|
+
# # If instance is a hash with the key :constant, the value can be
|
41
|
+
# # a global constant name containing a Faulty instance
|
42
|
+
# DB_FAULTY = Faulty.new
|
43
|
+
# Patch.circuit_from_hash(
|
44
|
+
# :mysql,
|
45
|
+
# { host: 'localhost', faulty: { instance: { constant: 'DB_FAULTY' } } }
|
46
|
+
# )
|
47
|
+
#
|
48
|
+
# @example
|
49
|
+
# # Certain patches may want to enforce certain options like :errors
|
50
|
+
# # This can be done via hash or the usual block syntax
|
51
|
+
# Patch.circuit_from_hash(:mysql,
|
52
|
+
# { host: 'localhost', faulty: {} }
|
53
|
+
# errors: [Mysql2::Error]
|
54
|
+
# )
|
55
|
+
#
|
56
|
+
# Patch.circuit_from_hash(:mysql,
|
57
|
+
# { host: 'localhost', faulty: {} }
|
58
|
+
# ) do |conf|
|
59
|
+
# conf.errors = [Mysql2::Error]
|
60
|
+
# end
|
61
|
+
#
|
62
|
+
# @param default_name [String] The default name for the circuit
|
63
|
+
# @param hash [Hash] A hash of user-provided options. Supports any circuit
|
64
|
+
# option and these additional options
|
65
|
+
# @option hash [String] :name The circuit name. Defaults to `default_name`
|
66
|
+
# @option hash [Boolean] :patch_errors By default, circuit errors will be
|
67
|
+
# subclasses of `options[:patched_error_module]`. The user can disable
|
68
|
+
# this by setting this option to false.
|
69
|
+
# @option hash [Faulty, String, Symbol, Hash{ constant: String }] :instance
|
70
|
+
# A reference to a faulty instance. See examples.
|
71
|
+
# @param options [Hash] Additional override options. Supports any circuit
|
72
|
+
# option and these additional ones.
|
73
|
+
# @option options [Module] :patched_error_module The namespace module
|
74
|
+
# for patched errors
|
75
|
+
# @yield [Circuit::Options] For setting override options in a block
|
76
|
+
# @return [Circuit, nil] The circuit if one was created
|
77
|
+
def circuit_from_hash(default_name, hash, **options, &block)
|
78
|
+
return unless hash
|
79
|
+
|
80
|
+
hash = symbolize_keys(hash)
|
81
|
+
name = hash.delete(:name) || default_name
|
82
|
+
patch_errors = hash.delete(:patch_errors) != false
|
83
|
+
error_module = options.delete(:patched_error_module)
|
84
|
+
hash[:error_module] ||= error_module if error_module && patch_errors
|
85
|
+
faulty = resolve_instance(hash.delete(:instance))
|
86
|
+
faulty.circuit(name, **hash, **options, &block)
|
87
|
+
end
|
88
|
+
|
89
|
+
# Create a full set of {CircuitError}s with a given base error class
|
90
|
+
#
|
91
|
+
# For patches that need their errors to be subclasses of a common base.
|
92
|
+
#
|
93
|
+
# @param namespace [Module] The module to define the error classes in
|
94
|
+
# @param base [Class] The base class for the error classes
|
95
|
+
# @return [void]
|
96
|
+
def define_circuit_errors(namespace, base)
|
97
|
+
circuit_error = Class.new(base) { include CircuitErrorBase }
|
98
|
+
namespace.const_set('CircuitError', circuit_error)
|
99
|
+
namespace.const_set('OpenCircuitError', Class.new(circuit_error))
|
100
|
+
namespace.const_set('CircuitFailureError', Class.new(circuit_error))
|
101
|
+
namespace.const_set('CircuitTrippedError', Class.new(circuit_error))
|
102
|
+
end
|
103
|
+
|
104
|
+
private
|
105
|
+
|
106
|
+
# Resolves a constant from a constant name or returns a default
|
107
|
+
#
|
108
|
+
# - If value is a string or symbol, gets a registered Faulty instance with that name
|
109
|
+
# - If value is a Hash with a key `:constant`, resolves the value to a global constant
|
110
|
+
# - If value is nil, gets Faulty.default
|
111
|
+
# - Otherwise, return value directly
|
112
|
+
#
|
113
|
+
# @param value [String, Symbol, Faulty, nil] The object or constant name to resolve
|
114
|
+
# @return [Object] The resolved Faulty instance
|
115
|
+
def resolve_instance(value)
|
116
|
+
case value
|
117
|
+
when String, Symbol
|
118
|
+
result = Faulty[value]
|
119
|
+
raise NameError, "No Faulty instance for #{value}" unless result
|
120
|
+
|
121
|
+
result
|
122
|
+
when Hash
|
123
|
+
const_name = value[:constant]
|
124
|
+
raise ArgumentError 'Missing hash key :constant for Faulty instance' unless const_name
|
125
|
+
|
126
|
+
Kernel.const_get(const_name)
|
127
|
+
when nil
|
128
|
+
Faulty.default
|
129
|
+
else
|
130
|
+
value
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
# Some config files may not suport symbol keys, so we convert the hash
|
135
|
+
# to use symbols so that users can pass in strings
|
136
|
+
#
|
137
|
+
# We cannot use transform_keys since we support Ruby < 2.5
|
138
|
+
#
|
139
|
+
# @param hash [Hash] A hash to convert
|
140
|
+
# @return [Hash] The hash with keys as symbols
|
141
|
+
def symbolize_keys(hash)
|
142
|
+
result = {}
|
143
|
+
hash.each do |key, val|
|
144
|
+
result[key.to_sym] = if val.is_a?(Hash)
|
145
|
+
symbolize_keys(val)
|
146
|
+
else
|
147
|
+
val
|
148
|
+
end
|
149
|
+
end
|
150
|
+
result
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|