breakers 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -3
- data/lib/breakers.rb +18 -4
- data/lib/breakers/client.rb +13 -1
- data/lib/breakers/outage.rb +59 -10
- data/lib/breakers/service.rb +63 -9
- data/lib/breakers/uptime_middleware.rb +6 -5
- data/lib/breakers/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a99dbaa5b10dc377c242f0707ca01e138e5119b5
|
4
|
+
data.tar.gz: f65d7cc872f4d76c2902362baab53c0a4825b73a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7a6fcda690f8eb2757cc3a82e5240c7b5d3cd3bec48a1e4f9c9e3bb32f2b7ae5c08b8b12c2215750187aac5623860f7e6b2aa0548c0b64fd9736d17ed1643aa2
|
7
|
+
data.tar.gz: 78e50c255536dcc2dfe3cb460197b4e1ac836cb7caa684341ac66e002b8f01927921eea3fb1011cc17f1ef6cc555b8126a26e047c35e25dcf45571fef96ceee9
|
data/README.md
CHANGED
@@ -31,7 +31,7 @@ service = Breakers::Service.new(
|
|
31
31
|
|
32
32
|
client = Breakers::Client.new(redis_connection: redis, services: [service])
|
33
33
|
|
34
|
-
Breakers.
|
34
|
+
Breakers.client = client
|
35
35
|
|
36
36
|
connection = Faraday.new do |conn|
|
37
37
|
conn.use :breakers
|
@@ -93,7 +93,7 @@ The logger should conform to Ruby's Logger API. See more information on plugins
|
|
93
93
|
The client can be configured globally with:
|
94
94
|
|
95
95
|
```ruby
|
96
|
-
Breakers.
|
96
|
+
Breakers.client = client
|
97
97
|
```
|
98
98
|
|
99
99
|
In a Rails app, it makes sense to create the services and client in an initializer and then apply them with this call. If you would like to
|
@@ -103,7 +103,7 @@ namespace the data in Redis with a prefix, you can make that happen with:
|
|
103
103
|
Breakers.redis_prefix = 'custom-'
|
104
104
|
```
|
105
105
|
|
106
|
-
The default prefix is
|
106
|
+
The default prefix is an empty string.
|
107
107
|
|
108
108
|
### Using the Middleware
|
109
109
|
|
@@ -140,6 +140,20 @@ end
|
|
140
140
|
|
141
141
|
It's ok for your plugin to implement only part of this interface.
|
142
142
|
|
143
|
+
### Forcing an Outage
|
144
|
+
|
145
|
+
Some services will have status endpoints that you can use to check their availability, and you may want to create an outage based on that.
|
146
|
+
Because this is a middleware, it doesn't have the ability to periodically check these endpoints, but you can add that type of check to your
|
147
|
+
application and then force an outage in breakers:
|
148
|
+
|
149
|
+
```ruby
|
150
|
+
service.begin_forced_outage!
|
151
|
+
service.end_forced_outage!
|
152
|
+
```
|
153
|
+
|
154
|
+
Unlike with outages detected by the middleware, forced outages are not periodically tested to see if they have completed and must be
|
155
|
+
manually ended with a call to `end_forced_outage!`.
|
156
|
+
|
143
157
|
### Redis Data Structure
|
144
158
|
|
145
159
|
Data is stored in Redis with the following structure:
|
data/lib/breakers.rb
CHANGED
@@ -6,24 +6,38 @@ require 'breakers/version'
|
|
6
6
|
|
7
7
|
require 'faraday'
|
8
8
|
|
9
|
+
# Implement the main module for the gem, which includes methods for global configuration
|
9
10
|
module Breakers
|
10
11
|
Faraday::Middleware.register_middleware(breakers: lambda { UptimeMiddleware })
|
11
12
|
|
12
|
-
#
|
13
|
-
|
13
|
+
# Set the global client for use in the middleware
|
14
|
+
#
|
15
|
+
# @param client [Breakers::Client] the client
|
16
|
+
def self.client=(client)
|
14
17
|
@client = client
|
15
18
|
end
|
16
|
-
# rubocop:enable Style/AccessorMethodName
|
17
19
|
|
20
|
+
# Return the global client
|
21
|
+
#
|
22
|
+
# @return [Breakers::Client] the client
|
18
23
|
def self.client
|
19
24
|
@client
|
20
25
|
end
|
21
26
|
|
27
|
+
# Breakers uses a number of Redis keys to store its data. You can pass an optional
|
28
|
+
# prefix here to use for the keys so that they will be namespaced properly. Note that
|
29
|
+
# it's also possible to create the Breakers::Client object with a Redis::Namespace
|
30
|
+
# object instead, in which case this is unnecessary.
|
31
|
+
#
|
32
|
+
# @param prefix [String] the prefix
|
22
33
|
def self.redis_prefix=(prefix)
|
23
34
|
@redis_prefix = prefix
|
24
35
|
end
|
25
36
|
|
37
|
+
# Query for the Redis key prefix
|
38
|
+
#
|
39
|
+
# @return [String] the prefix
|
26
40
|
def self.redis_prefix
|
27
|
-
@redis_prefix || '
|
41
|
+
@redis_prefix || ''
|
28
42
|
end
|
29
43
|
end
|
data/lib/breakers/client.rb
CHANGED
@@ -1,10 +1,18 @@
|
|
1
1
|
module Breakers
|
2
|
+
# The client contains all of the data required to operate Breakers. Creating one and
|
3
|
+
# setting it as the global client allows the middleware to operate without parameters
|
2
4
|
class Client
|
3
5
|
attr_reader :services
|
4
6
|
attr_reader :plugins
|
5
7
|
attr_reader :redis_connection
|
6
8
|
attr_reader :logger
|
7
9
|
|
10
|
+
# Create the Client object.
|
11
|
+
#
|
12
|
+
# @param redis_connection [Redis] the Redis connection or namespace to use
|
13
|
+
# @param services [Breakers::Service] a list of services to be monitored
|
14
|
+
# @param plugins [Object] a list of plugins to call as events occur
|
15
|
+
# @param logger [Logger] a logger implementing the Ruby Logger interface to call as events occur
|
8
16
|
def initialize(redis_connection:, services:, plugins: nil, logger: nil)
|
9
17
|
@redis_connection = redis_connection
|
10
18
|
@services = Array(services)
|
@@ -12,9 +20,13 @@ module Breakers
|
|
12
20
|
@logger = logger
|
13
21
|
end
|
14
22
|
|
23
|
+
# Given a request environment, return the service that should handle it.
|
24
|
+
#
|
25
|
+
# @param request_env [Faraday::Env] the request environment
|
26
|
+
# @return [Breakers::Service] the service object
|
15
27
|
def service_for_request(request_env:)
|
16
28
|
@services.find do |service|
|
17
|
-
service.handles_request?(request_env)
|
29
|
+
service.handles_request?(request_env: request_env)
|
18
30
|
end
|
19
31
|
end
|
20
32
|
end
|
data/lib/breakers/outage.rb
CHANGED
@@ -1,77 +1,126 @@
|
|
1
1
|
require 'multi_json'
|
2
2
|
|
3
3
|
module Breakers
|
4
|
+
# A class defining an outage on a service
|
4
5
|
class Outage
|
5
6
|
attr_reader :service
|
6
7
|
attr_reader :body
|
7
8
|
|
8
|
-
|
9
|
+
# Return the most recent outage on the given service
|
10
|
+
#
|
11
|
+
# @param service [Breakers::Service] the service to look in
|
12
|
+
# @return [Breakers::Outage] the most recent outage, or nil
|
13
|
+
def self.find_latest(service:)
|
9
14
|
data = Breakers.client.redis_connection.zrange(outages_key(service: service), -1, -1)[0]
|
10
|
-
data && new(service: service,
|
15
|
+
data && new(service: service, body: data)
|
11
16
|
end
|
12
17
|
|
18
|
+
# Return all of the outages on the given service that begin in the time range
|
19
|
+
#
|
20
|
+
# @param service [Breakers::Service] the service to look in
|
21
|
+
# @param start_time [Time] the beginning of the time range
|
22
|
+
# @param end_time [Time] the end of the time range
|
23
|
+
# @return [Breakers::Outage] a list of the outages in the range
|
13
24
|
def self.in_range(service:, start_time:, end_time:)
|
14
25
|
data = Breakers.client.redis_connection.zrangebyscore(
|
15
26
|
outages_key(service: service),
|
16
27
|
start_time.to_i,
|
17
28
|
end_time.to_i
|
18
29
|
)
|
19
|
-
data.map { |item| new(service: service,
|
30
|
+
data.map { |item| new(service: service, body: item) }
|
20
31
|
end
|
21
32
|
|
22
|
-
|
23
|
-
|
33
|
+
# Create a new outage on the given service
|
34
|
+
#
|
35
|
+
# @param service [Breakers::Service] the service to create it for
|
36
|
+
# @param forced [Boolean] is the service forced, or created via the middleware
|
37
|
+
# @return [Breakers::Outage] the new outage
|
38
|
+
def self.create(service:, forced: false)
|
39
|
+
data = MultiJson.dump(start_time: Time.now.utc.to_i, forced: forced)
|
24
40
|
Breakers.client.redis_connection.zadd(outages_key(service: service), Time.now.utc.to_i, data)
|
25
41
|
|
26
|
-
Breakers.client.logger&.error(msg: 'Breakers outage beginning', service: service.name)
|
42
|
+
Breakers.client.logger&.error(msg: 'Breakers outage beginning', service: service.name, forced: forced)
|
27
43
|
|
28
44
|
Breakers.client.plugins.each do |plugin|
|
29
|
-
plugin.on_outage_begin(Outage.new(service: service,
|
45
|
+
plugin.on_outage_begin(Outage.new(service: service, body: data)) if plugin.respond_to?(:on_outage_begin)
|
30
46
|
end
|
31
47
|
end
|
32
48
|
|
49
|
+
# Get the key for storing the outage data in Redis for this service
|
50
|
+
#
|
51
|
+
# @param service [Breakers::Service] the service
|
52
|
+
# @return [String] the Redis key
|
33
53
|
def self.outages_key(service:)
|
34
54
|
"#{Breakers.redis_prefix}#{service.name}-outages"
|
35
55
|
end
|
36
56
|
|
37
|
-
|
38
|
-
|
57
|
+
# Create a new outage
|
58
|
+
#
|
59
|
+
# @param service [Breakers::Service] the service the outage is for
|
60
|
+
# @param body [Hash] the data to store in the outage, with keys start_time, end_time, last_test_time, and forced
|
61
|
+
# @return [Breakers::Outage] the new outage
|
62
|
+
def initialize(service:, body:)
|
63
|
+
@body = MultiJson.load(body)
|
39
64
|
@service = service
|
40
65
|
end
|
41
66
|
|
67
|
+
# Check to see if the outage has ended
|
68
|
+
#
|
69
|
+
# @return [Boolean] the status
|
42
70
|
def ended?
|
43
71
|
@body.key?('end_time')
|
44
72
|
end
|
45
73
|
|
74
|
+
# Was the outage forced?
|
75
|
+
#
|
76
|
+
# @return [Boolean] the status
|
77
|
+
def forced?
|
78
|
+
@body['forced']
|
79
|
+
end
|
80
|
+
|
81
|
+
# Tell the outage to end, which will allow requests to begin flowing again
|
46
82
|
def end!
|
47
83
|
new_body = @body.dup
|
48
84
|
new_body['end_time'] = Time.now.utc.to_i
|
49
85
|
replace_body(body: new_body)
|
50
86
|
|
51
|
-
Breakers.client.logger&.info(msg: 'Breakers outage ending', service: @service.name)
|
87
|
+
Breakers.client.logger&.info(msg: 'Breakers outage ending', service: @service.name, forced: forced?)
|
52
88
|
Breakers.client.plugins.each do |plugin|
|
53
89
|
plugin.on_outage_end(self) if plugin.respond_to?(:on_outage_begin)
|
54
90
|
end
|
55
91
|
end
|
56
92
|
|
93
|
+
# Get the time at which the outage started
|
94
|
+
#
|
95
|
+
# @return [Time] the time
|
57
96
|
def start_time
|
58
97
|
@body['start_time'] && Time.at(@body['start_time']).utc
|
59
98
|
end
|
60
99
|
|
100
|
+
# Get the time at which the outage ended
|
101
|
+
#
|
102
|
+
# @return [Time] the time
|
61
103
|
def end_time
|
62
104
|
@body['end_time'] && Time.at(@body['end_time']).utc
|
63
105
|
end
|
64
106
|
|
107
|
+
# Get the time at which the outage last received a new request
|
108
|
+
#
|
109
|
+
# @return [Time] the time
|
65
110
|
def last_test_time
|
66
111
|
(@body['last_test_time'] && Time.at(@body['last_test_time']).utc) || start_time
|
67
112
|
end
|
68
113
|
|
114
|
+
# Update the last test time to now
|
69
115
|
def update_last_test_time!
|
70
116
|
new_body = @body.dup
|
71
117
|
new_body['last_test_time'] = Time.now.utc.to_i
|
72
118
|
replace_body(body: new_body)
|
73
119
|
end
|
74
120
|
|
121
|
+
# Check to see if the outage should be retested to make sure it's still ongoing
|
122
|
+
#
|
123
|
+
# @return [Boolean] is it ready?
|
75
124
|
def ready_for_retest?(wait_seconds:)
|
76
125
|
(Time.now.utc - last_test_time) > wait_seconds
|
77
126
|
end
|
data/lib/breakers/service.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
module Breakers
|
2
|
+
# A service defines a backend system that your application relies upon. This class
|
3
|
+
# allows you to configure the outage detection for a service as well as to define which
|
4
|
+
# requests belong to it.
|
2
5
|
class Service
|
3
6
|
DEFAULT_OPTS = {
|
4
7
|
seconds_before_retry: 60,
|
@@ -6,35 +9,74 @@ module Breakers
|
|
6
9
|
data_retention_seconds: 60 * 60 * 24 * 30
|
7
10
|
}.freeze
|
8
11
|
|
12
|
+
# Create a new service
|
13
|
+
#
|
14
|
+
# @param [Hash] opts the options to create the service with
|
15
|
+
# @option opts [String] :name The name of the service for reporting and logging purposes
|
16
|
+
# @option opts [Proc] :request_matcher A proc taking a Faraday::Env as an argument that returns true if the service handles that request
|
17
|
+
# @option opts [Integer] :seconds_before_retry The number of seconds to wait after an outage begins before testing with a new request
|
18
|
+
# @option opts [Integer] :error_threshold The percentage of errors over the last two minutes that indicates an outage
|
19
|
+
# @option opts [Integer] :data_retention_seconds The number of seconds to retain success and error data in Redis
|
9
20
|
def initialize(opts)
|
10
21
|
@configuration = DEFAULT_OPTS.merge(opts)
|
11
22
|
end
|
12
23
|
|
24
|
+
# Get the name of the service
|
25
|
+
#
|
26
|
+
# @return [String] the name
|
13
27
|
def name
|
14
28
|
@configuration[:name]
|
15
29
|
end
|
16
30
|
|
17
|
-
|
31
|
+
# Given a Faraday::Env, return true if this service handles the request, via its matcher
|
32
|
+
#
|
33
|
+
# @param request_env [Faraday::Env] the request environment
|
34
|
+
# @return [Boolean] should the service handle the request
|
35
|
+
def handles_request?(request_env:)
|
18
36
|
@configuration[:request_matcher].call(request_env)
|
19
37
|
end
|
20
38
|
|
39
|
+
# Get the seconds before retry parameter
|
40
|
+
#
|
41
|
+
# @return [Integer] the value
|
21
42
|
def seconds_before_retry
|
22
43
|
@configuration[:seconds_before_retry]
|
23
44
|
end
|
24
45
|
|
46
|
+
# Indicate that an error has occurred and potentially create an outage
|
25
47
|
def add_error
|
26
48
|
increment_key(key: errors_key)
|
27
49
|
maybe_create_outage
|
28
50
|
end
|
29
51
|
|
52
|
+
# Indicate that a successful response has occurred
|
30
53
|
def add_success
|
31
54
|
increment_key(key: successes_key)
|
32
55
|
end
|
33
56
|
|
34
|
-
|
35
|
-
|
57
|
+
# Force an outage to begin on the service. Forced outages are not periodically retested.
|
58
|
+
def begin_forced_outage!
|
59
|
+
Outage.create(service: self, forced: true)
|
36
60
|
end
|
37
61
|
|
62
|
+
# End a forced outage on the service.
|
63
|
+
def end_forced_outage!
|
64
|
+
latest = Outage.find_latest(service: self)
|
65
|
+
if latest.forced?
|
66
|
+
latest.end!
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Return the most recent outage on the service
|
71
|
+
def latest_outage
|
72
|
+
Outage.find_latest(service: self)
|
73
|
+
end
|
74
|
+
|
75
|
+
# Return a list of all outages in the given time range
|
76
|
+
#
|
77
|
+
# @param start_time [Time] the beginning of the range
|
78
|
+
# @param end_time [Time] the end of the range
|
79
|
+
# @return [Array[Outage]] a list of outages that began in the range
|
38
80
|
def outages_in_range(start_time:, end_time:)
|
39
81
|
Outage.in_range(
|
40
82
|
service: self,
|
@@ -43,12 +85,24 @@ module Breakers
|
|
43
85
|
)
|
44
86
|
end
|
45
87
|
|
46
|
-
|
47
|
-
|
88
|
+
# Return data about the successful request counts in the time range
|
89
|
+
#
|
90
|
+
# @param start_time [Time] the beginning of the range
|
91
|
+
# @param end_time [Time] the end of the range
|
92
|
+
# @param sample_minutes [Integer] the rate at which to sample the data
|
93
|
+
# @return [Array[Hash]] a list of hashes in the form: { count: Integer, time: Unix Timestamp }
|
94
|
+
def successes_in_range(start_time:, end_time:, sample_minutes: 60)
|
95
|
+
values_in_range(start_time: start_time, end_time: end_time, type: :successes, sample_minutes: sample_minutes)
|
48
96
|
end
|
49
97
|
|
50
|
-
|
51
|
-
|
98
|
+
# Return data about the failed request counts in the time range
|
99
|
+
#
|
100
|
+
# @param start_time [Time] the beginning of the range
|
101
|
+
# @param end_time [Time] the end of the range
|
102
|
+
# @param sample_minutes [Integer] the rate at which to sample the data
|
103
|
+
# @return [Array[Hash]] a list of hashes in the form: { count: Integer, time: Unix Timestamp }
|
104
|
+
def errors_in_range(start_time:, end_time:, sample_minutes: 60)
|
105
|
+
values_in_range(start_time: start_time, end_time: end_time, type: :errors, sample_minutes: sample_minutes)
|
52
106
|
end
|
53
107
|
|
54
108
|
protected
|
@@ -61,7 +115,7 @@ module Breakers
|
|
61
115
|
"#{Breakers.redis_prefix}#{name}-successes-#{align_time_on_minute(time: time).to_i}"
|
62
116
|
end
|
63
117
|
|
64
|
-
def values_in_range(start_time:, end_time:, type:,
|
118
|
+
def values_in_range(start_time:, end_time:, type:, sample_minutes:)
|
65
119
|
start_time = align_time_on_minute(time: start_time)
|
66
120
|
end_time = align_time_on_minute(time: end_time)
|
67
121
|
keys = []
|
@@ -73,7 +127,7 @@ module Breakers
|
|
73
127
|
elsif type == :successes
|
74
128
|
keys << successes_key(time: start_time)
|
75
129
|
end
|
76
|
-
start_time +=
|
130
|
+
start_time += sample_minutes * 60
|
77
131
|
end
|
78
132
|
Breakers.client.redis_connection.mget(keys).each_with_index.map do |value, idx|
|
79
133
|
{ count: value.to_i, time: times[idx] }
|
@@ -2,6 +2,7 @@ require 'faraday'
|
|
2
2
|
require 'multi_json'
|
3
3
|
|
4
4
|
module Breakers
|
5
|
+
# The faraday middleware
|
5
6
|
class UptimeMiddleware < Faraday::Middleware
|
6
7
|
def initialize(app)
|
7
8
|
super(app)
|
@@ -14,13 +15,13 @@ module Breakers
|
|
14
15
|
return @app.call(request_env)
|
15
16
|
end
|
16
17
|
|
17
|
-
|
18
|
+
latest_outage = service.latest_outage
|
18
19
|
|
19
|
-
if
|
20
|
-
if
|
21
|
-
handle_request(service: service, request_env: request_env, current_outage:
|
20
|
+
if latest_outage && !latest_outage.ended?
|
21
|
+
if latest_outage.ready_for_retest?(wait_seconds: service.seconds_before_retry)
|
22
|
+
handle_request(service: service, request_env: request_env, current_outage: latest_outage)
|
22
23
|
else
|
23
|
-
outage_response(outage:
|
24
|
+
outage_response(outage: latest_outage, service: service)
|
24
25
|
end
|
25
26
|
else
|
26
27
|
handle_request(service: service, request_env: request_env)
|
data/lib/breakers/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: breakers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aubrey Holland
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-10-
|
11
|
+
date: 2016-10-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|
@@ -220,3 +220,4 @@ signing_key:
|
|
220
220
|
specification_version: 4
|
221
221
|
summary: Handle outages to backend systems with a Faraday middleware
|
222
222
|
test_files: []
|
223
|
+
has_rdoc:
|