kruby-controller 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +201 -0
- data/README.md +117 -0
- data/Rakefile +8 -0
- data/exe/kruby-controller +32 -0
- data/lib/kruby-controller.rb +8 -0
- data/lib/kruby_controller/builder/controller_builder.rb +52 -0
- data/lib/kruby_controller/cache/cache.rb +100 -0
- data/lib/kruby_controller/cache/deep_copy.rb +32 -0
- data/lib/kruby_controller/cache/delta.rb +12 -0
- data/lib/kruby_controller/cache/informer.rb +453 -0
- data/lib/kruby_controller/cache/request_error.rb +55 -0
- data/lib/kruby_controller/cache/store.rb +92 -0
- data/lib/kruby_controller/cache/watch_stream.rb +144 -0
- data/lib/kruby_controller/cli/create_api_command.rb +335 -0
- data/lib/kruby_controller/cli/flag_parser.rb +85 -0
- data/lib/kruby_controller/cli/init_command.rb +383 -0
- data/lib/kruby_controller/controller.rb +200 -0
- data/lib/kruby_controller/crd.rb +141 -0
- data/lib/kruby_controller/election/leader_elector.rb +212 -0
- data/lib/kruby_controller/finalizer/finalizer_manager.rb +31 -0
- data/lib/kruby_controller/generator/crd_generator.rb +109 -0
- data/lib/kruby_controller/gvk.rb +59 -0
- data/lib/kruby_controller/handler/enqueued_object.rb +18 -0
- data/lib/kruby_controller/handler/enqueued_owner.rb +51 -0
- data/lib/kruby_controller/handler/predicated.rb +21 -0
- data/lib/kruby_controller/kubernetes_object_client.rb +77 -0
- data/lib/kruby_controller/logger.rb +11 -0
- data/lib/kruby_controller/manager.rb +291 -0
- data/lib/kruby_controller/owner/owner_reference.rb +23 -0
- data/lib/kruby_controller/queue/rate_limiter.rb +51 -0
- data/lib/kruby_controller/queue/work_queue.rb +375 -0
- data/lib/kruby_controller/reconciler.rb +18 -0
- data/lib/kruby_controller/request.rb +59 -0
- data/lib/kruby_controller/result.rb +50 -0
- data/lib/kruby_controller/status/status_updater.rb +29 -0
- data/lib/kruby_controller/version.rb +5 -0
- data/lib/kruby_controller.rb +87 -0
- metadata +138 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module KrubyController
|
|
6
|
+
# Adapter over a raw Kubernetes::ApiClient that exposes the two write verbs the
|
|
7
|
+
# helper modules (FinalizerManager / StatusUpdater) need: a full-object
|
|
8
|
+
# replace (`update`) and a `/status` subresource patch (`patch_status`).
|
|
9
|
+
#
|
|
10
|
+
# It exists because kruby's CustomObjectsApi does not offer a PUT `update` and
|
|
11
|
+
# its `patch_*` methods are hardwired to `application/json-patch+json` (a
|
|
12
|
+
# JSON-Patch *array* body), which cannot carry a full object. Building the
|
|
13
|
+
# requests directly on the ApiClient (the same path Informer uses) lets us
|
|
14
|
+
# pick the correct verb and content type per operation.
|
|
15
|
+
#
|
|
16
|
+
# Pass an instance as the `client:` for FinalizerManager / StatusUpdater:
|
|
17
|
+
#
|
|
18
|
+
# api_client = Kubernetes::ApiClient.new(config)
|
|
19
|
+
# client = KrubyController::KubernetesObjectClient.new(api_client: api_client, gvk: gvk)
|
|
20
|
+
# fm = KrubyController::Finalizer::FinalizerManager.new(client: client, gvk: gvk)
|
|
21
|
+
# su = KrubyController::Status::StatusUpdater.new(client: client, gvk: gvk)
|
|
22
|
+
class KubernetesObjectClient
|
|
23
|
+
def initialize(api_client:, gvk:)
|
|
24
|
+
@api_client = api_client
|
|
25
|
+
@gvk = gvk
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Replace the object on the server (PUT, application/json).
|
|
29
|
+
def update(obj)
|
|
30
|
+
send_object("PUT", object_path(obj), obj, "application/json")
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Patch only the `/status` subresource (PATCH, merge-patch).
|
|
34
|
+
def patch_status(obj)
|
|
35
|
+
send_object("PATCH", "#{object_path(obj)}/status", obj, "application/merge-patch+json")
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def object_path(obj)
|
|
41
|
+
name = obj.dig("metadata", "name")
|
|
42
|
+
raise KrubyController::Error, "object has no metadata.name" if name.nil? || name.empty?
|
|
43
|
+
|
|
44
|
+
base = @gvk.group.to_s.empty? ? "/api" : "/apis/#{@gvk.group}"
|
|
45
|
+
[base, @gvk.version, namespace_segment(obj, name), @gvk.resource, name].compact.join("/")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def namespace_segment(obj, name)
|
|
49
|
+
return nil unless @gvk.namespaced?
|
|
50
|
+
|
|
51
|
+
namespace = obj.dig("metadata", "namespace")
|
|
52
|
+
if namespace.nil? || namespace.empty?
|
|
53
|
+
raise KrubyController::Error, "namespaced object '#{name}' has no metadata.namespace"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
"namespaces/#{namespace}"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def send_object(method, path, obj, content_type)
|
|
60
|
+
request = @api_client.build_request(
|
|
61
|
+
method,
|
|
62
|
+
path,
|
|
63
|
+
header_params: { "Content-Type" => content_type },
|
|
64
|
+
body: obj.to_json,
|
|
65
|
+
auth_names: ["BearerToken"]
|
|
66
|
+
)
|
|
67
|
+
response = request.run
|
|
68
|
+
unless response.success?
|
|
69
|
+
raise KrubyController::Error,
|
|
70
|
+
"#{method} #{path} failed (HTTP #{response.code}): #{response.status_message}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
body = response.body
|
|
74
|
+
body.nil? || body.empty? ? nil : JSON.parse(body)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module KrubyController
|
|
4
|
+
class Manager
|
|
5
|
+
attr_reader :cache, :stopped
|
|
6
|
+
|
|
7
|
+
# Interval at which the running thread polls for a stop request (the
|
|
8
|
+
# signal flag, a programmatic stop, or a startup failure).
|
|
9
|
+
SIGNAL_POLL_INTERVAL = 0.1
|
|
10
|
+
|
|
11
|
+
def initialize(
|
|
12
|
+
kube_config: nil,
|
|
13
|
+
namespace: nil,
|
|
14
|
+
leader_elect: true,
|
|
15
|
+
lease_name: "kruby-controller-lock",
|
|
16
|
+
lease_namespace: "kube-system",
|
|
17
|
+
lease_duration: 15,
|
|
18
|
+
renew_deadline: 10,
|
|
19
|
+
retry_period: 2,
|
|
20
|
+
stop_timeout: 30,
|
|
21
|
+
logger: nil,
|
|
22
|
+
list_page_size: nil
|
|
23
|
+
)
|
|
24
|
+
@kube_config = kube_config
|
|
25
|
+
@namespace = namespace
|
|
26
|
+
@leader_elect = leader_elect
|
|
27
|
+
@lease_opts = { lease_name:, lease_namespace:, lease_duration:, renew_deadline:, retry_period: }
|
|
28
|
+
@stop_timeout = stop_timeout
|
|
29
|
+
@logger = logger
|
|
30
|
+
@cache = Cache::SharedCache.new(kube_config:, namespace:, list_page_size:)
|
|
31
|
+
@controllers = []
|
|
32
|
+
@stopped = false
|
|
33
|
+
@stop_requested_by_signal = false
|
|
34
|
+
@stop_channel = Channel.new
|
|
35
|
+
@startup_error = nil
|
|
36
|
+
KrubyController.logger = @logger if @logger
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def add(controller_or_reconciler, **options)
|
|
40
|
+
reconciler = instantiate_reconciler(controller_or_reconciler)
|
|
41
|
+
gvk = resolve_gvk(options)
|
|
42
|
+
controller = Controller.new(
|
|
43
|
+
reconciler:,
|
|
44
|
+
cache: @cache,
|
|
45
|
+
for_gvk: gvk,
|
|
46
|
+
max_concurrent: options.fetch(:max_concurrent, 1),
|
|
47
|
+
name: options[:name],
|
|
48
|
+
owns: Array(options.fetch(:owns, [])).map { |target| resolve_target(target, options) },
|
|
49
|
+
watches: Array(options.fetch(:watches, [])).map do |w|
|
|
50
|
+
w[:gvk] ? w.merge(gvk: resolve_target(w[:gvk], options)) : w
|
|
51
|
+
end,
|
|
52
|
+
predicate: options[:predicate],
|
|
53
|
+
rate_limiter: options[:rate_limiter]
|
|
54
|
+
)
|
|
55
|
+
@controllers << controller
|
|
56
|
+
controller
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def for(gvk)
|
|
60
|
+
Builder::ControllerBuilder.new(self, gvk)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def start
|
|
64
|
+
KrubyController.logger.info("Starting manager...")
|
|
65
|
+
|
|
66
|
+
# Both modes block until stop is requested: SIGTERM/SIGINT set a flag
|
|
67
|
+
# (trap handlers must not acquire Mutexes) and the running thread
|
|
68
|
+
# performs the actual stop from normal context.
|
|
69
|
+
setup_signal_handlers
|
|
70
|
+
|
|
71
|
+
if @leader_elect
|
|
72
|
+
@leader_elector = Election::LeaderElector.new(
|
|
73
|
+
client: coordination_client,
|
|
74
|
+
on_stopped_leading: -> { stop_components },
|
|
75
|
+
**@lease_opts
|
|
76
|
+
)
|
|
77
|
+
@leader_elector.run { start_leading }
|
|
78
|
+
wait_for_stop
|
|
79
|
+
# Safe from this thread (a different thread than the elector's
|
|
80
|
+
# election_loop), so #stop's join cannot deadlock. Stopping first
|
|
81
|
+
# releases the lease and halts renewal before we fail startup.
|
|
82
|
+
@leader_elector&.stop
|
|
83
|
+
raise @startup_error if @startup_error
|
|
84
|
+
else
|
|
85
|
+
start_and_sync
|
|
86
|
+
wait_for_stop
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Startup work under leadership. On a fatal startup failure (the cache
|
|
91
|
+
# cannot sync), record the error and signal the stop channel so
|
|
92
|
+
# Manager#start fails with the real cause instead of keeping leadership
|
|
93
|
+
# with running controllers and an inert informer. start_and_sync has
|
|
94
|
+
# already stopped the components before re-raising.
|
|
95
|
+
def start_leading
|
|
96
|
+
start_and_sync
|
|
97
|
+
rescue StandardError => e
|
|
98
|
+
@startup_error = e
|
|
99
|
+
@stop_channel.send(:stopped)
|
|
100
|
+
raise
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def stop(timeout: @stop_timeout)
|
|
104
|
+
return if @stopped
|
|
105
|
+
|
|
106
|
+
@stopped = true
|
|
107
|
+
KrubyController.logger.info("Stopping manager...")
|
|
108
|
+
@leader_elector&.stop
|
|
109
|
+
# The drain budget is split evenly across controllers so the total
|
|
110
|
+
# reconcile wait stays bounded by `timeout`: each controller may use up
|
|
111
|
+
# to its share, and a worker still inside a reconcile after its share
|
|
112
|
+
# is abandoned (it exits on its next loop check; the closed queue
|
|
113
|
+
# delivers no further work). Manager#stop returns per contract even
|
|
114
|
+
# when a reconcile or watch does not respond.
|
|
115
|
+
budget = drain_budget
|
|
116
|
+
@controllers.each { |c| c.stop(timeout: budget) }
|
|
117
|
+
@cache.stop
|
|
118
|
+
@stop_channel.send(:stopped)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def leader?
|
|
122
|
+
return true unless @leader_elect
|
|
123
|
+
|
|
124
|
+
@leader_elector&.leader || false
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
private
|
|
128
|
+
|
|
129
|
+
def instantiate_reconciler(controller_or_reconciler)
|
|
130
|
+
if controller_or_reconciler.is_a?(Class) && controller_or_reconciler < Reconciler
|
|
131
|
+
controller_or_reconciler.new
|
|
132
|
+
else
|
|
133
|
+
controller_or_reconciler
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def resolve_gvk(options)
|
|
138
|
+
resolve_target(options[:for] || options[:for_gvk], options)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def resolve_target(target, options = {})
|
|
142
|
+
case target
|
|
143
|
+
when GVK
|
|
144
|
+
target
|
|
145
|
+
when Class
|
|
146
|
+
GVK.new(
|
|
147
|
+
group: target.group,
|
|
148
|
+
version: target.version,
|
|
149
|
+
kind: target.kind,
|
|
150
|
+
resource: target.respond_to?(:plural) ? target.plural : nil,
|
|
151
|
+
namespaced: target.respond_to?(:namespaced) ? target.namespaced : options.fetch(:namespaced, true)
|
|
152
|
+
)
|
|
153
|
+
when String, Symbol
|
|
154
|
+
group = options[:group]
|
|
155
|
+
version = options[:version]
|
|
156
|
+
unless group && version
|
|
157
|
+
raise ArgumentError,
|
|
158
|
+
"a bare Kind (#{target}) cannot be resolved; pass a GVK, a CRD class, " \
|
|
159
|
+
"or provide group:/version: options"
|
|
160
|
+
end
|
|
161
|
+
GVK.new(group:, version:, kind: target.to_s, namespaced: options.fetch(:namespaced, true))
|
|
162
|
+
else
|
|
163
|
+
raise ArgumentError, "for:/for_gvk:/owns:/watches: must be a KrubyController::GVK, a CRD class, or a Kind"
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def start_and_sync
|
|
168
|
+
# A stop requested before startup began: register nothing, start nothing.
|
|
169
|
+
return if @stopped
|
|
170
|
+
|
|
171
|
+
register_controllers
|
|
172
|
+
@cache.start
|
|
173
|
+
# A cache that is not synced within the timeout is just as fatal as one
|
|
174
|
+
# that fails outright: fail startup (stopping the components we just
|
|
175
|
+
# started) instead of running controllers against an empty/stale cache.
|
|
176
|
+
synced = @cache.wait_for_sync
|
|
177
|
+
if @stopped || !leader?
|
|
178
|
+
# A stop raced in during the sync wait, or leadership was lost while
|
|
179
|
+
# the cache was syncing (the elector flags the loss before its
|
|
180
|
+
# on_stopped_leading cleanup runs). stop() may already have stopped
|
|
181
|
+
# the components; stop_components is idempotent. Workers must never
|
|
182
|
+
# start without the lease or on a stopped manager.
|
|
183
|
+
stop_components
|
|
184
|
+
return
|
|
185
|
+
end
|
|
186
|
+
raise KrubyController::Error, "Manager: cache did not sync within the timeout" unless synced
|
|
187
|
+
|
|
188
|
+
# Workers start only after every informer completed its initial List
|
|
189
|
+
# (FR-03). Events observed during the wait are held in the work queue
|
|
190
|
+
# and replayed once workers run, so no reconcile sees a partially
|
|
191
|
+
# synced cache and no initial event is lost.
|
|
192
|
+
start_controllers
|
|
193
|
+
rescue StandardError => e
|
|
194
|
+
# A cache that cannot sync is fatal: stop the components we just
|
|
195
|
+
# started and let the error propagate (Manager#start fails with it).
|
|
196
|
+
KrubyController.logger.error("Manager: cache sync failed: #{e.message}; stopping components")
|
|
197
|
+
stop_components
|
|
198
|
+
raise
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def stop_components
|
|
202
|
+
# The leader-election stop path (on_stopped_leading inside the
|
|
203
|
+
# elector's thread) and the startup-failure rescue share this drain:
|
|
204
|
+
# it must use the same bounded budget as Manager#stop, or an
|
|
205
|
+
# unresponsive reconcile would block the elector join — and with it
|
|
206
|
+
# Manager#stop — beyond stop_timeout in the default mode.
|
|
207
|
+
budget = drain_budget
|
|
208
|
+
@controllers.each { |c| c.stop(timeout: budget) }
|
|
209
|
+
@cache.stop
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def drain_budget
|
|
213
|
+
@stop_timeout.to_f / [@controllers.size, 1].max
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def start_controllers
|
|
217
|
+
@controllers.each(&:start)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def register_controllers
|
|
221
|
+
@controllers.each(&:register)
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def setup_signal_handlers
|
|
225
|
+
%w[TERM INT].each do |sig|
|
|
226
|
+
# A trap handler must not acquire Mutexes (raises ThreadError in trap
|
|
227
|
+
# context); only record the request. The running thread observes the
|
|
228
|
+
# flag in #wait_for_stop and performs the actual stop there.
|
|
229
|
+
Signal.trap(sig) { @stop_requested_by_signal = true }
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Blocks until the manager is stopped AND the stop's own cleanup has
|
|
234
|
+
# completed: #stop sets @stopped before draining, so observing only the
|
|
235
|
+
# flag would let this thread return while another thread is still
|
|
236
|
+
# draining (a main-thread start returning early can kill the drainer).
|
|
237
|
+
# Either #stop's channel send (or the startup-failure signal) ends the
|
|
238
|
+
# wait; a signal received here performs #stop first, which sends on the
|
|
239
|
+
# channel when its drain finishes.
|
|
240
|
+
def wait_for_stop
|
|
241
|
+
loop do
|
|
242
|
+
stop if @stop_requested_by_signal
|
|
243
|
+
return @stop_channel.receive if @stopped
|
|
244
|
+
return if @stop_channel.received?
|
|
245
|
+
|
|
246
|
+
sleep SIGNAL_POLL_INTERVAL
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def coordination_client
|
|
251
|
+
@coordination_client ||= begin
|
|
252
|
+
config = if @kube_config
|
|
253
|
+
cfg = Kubernetes::Configuration.new
|
|
254
|
+
Kubernetes::KubeConfig.new(@kube_config).configure(cfg)
|
|
255
|
+
cfg
|
|
256
|
+
else
|
|
257
|
+
Kubernetes::Configuration.default_config
|
|
258
|
+
end
|
|
259
|
+
Kubernetes::CoordinationV1Api.new(Kubernetes::ApiClient.new(config))
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
class Channel
|
|
264
|
+
def initialize
|
|
265
|
+
@mutex = Mutex.new
|
|
266
|
+
@cond = ConditionVariable.new
|
|
267
|
+
@value = nil
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def send(value = nil)
|
|
271
|
+
@mutex.synchronize do
|
|
272
|
+
@value = value
|
|
273
|
+
@cond.signal
|
|
274
|
+
end
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def receive
|
|
278
|
+
@mutex.synchronize do
|
|
279
|
+
@cond.wait(@mutex) until @value
|
|
280
|
+
@value
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# Non-blocking check used by #wait_for_stop, which cannot block on
|
|
285
|
+
# #receive and also poll the signal flag in the same thread.
|
|
286
|
+
def received?
|
|
287
|
+
@mutex.synchronize { !@value.nil? }
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module KrubyController
|
|
4
|
+
module Owner
|
|
5
|
+
module OwnerReference
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def set_owner(object, owner, controller: false, block_owner_deletion: false)
|
|
9
|
+
ref = {
|
|
10
|
+
"apiVersion" => owner["apiVersion"],
|
|
11
|
+
"kind" => owner["kind"],
|
|
12
|
+
"name" => owner.dig("metadata", "name"),
|
|
13
|
+
"uid" => owner.dig("metadata", "uid"),
|
|
14
|
+
}
|
|
15
|
+
ref["controller"] = controller if controller
|
|
16
|
+
ref["blockOwnerDeletion"] = block_owner_deletion if block_owner_deletion
|
|
17
|
+
object["metadata"]["ownerReferences"] ||= []
|
|
18
|
+
object["metadata"]["ownerReferences"] << ref
|
|
19
|
+
object
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module KrubyController
|
|
4
|
+
module Queue
|
|
5
|
+
# Per-key exponential backoff for failed reconciles, mirroring
|
|
6
|
+
# controller-runtime's ItemExponentialFailureRateLimiter: the n-th
|
|
7
|
+
# consecutive failure for a key waits base * 2**(n-1) seconds, capped at
|
|
8
|
+
# max. Keys are independent: a failing key never slows down others.
|
|
9
|
+
#
|
|
10
|
+
# limiter = ExponentialBackoffRateLimiter.new(base: 5.0, max: 300.0)
|
|
11
|
+
# limiter.backoff("default/foo") # => 5.0
|
|
12
|
+
# limiter.backoff("default/foo") # => 10.0
|
|
13
|
+
# limiter.forget("default/foo") # successful reconcile resets the count
|
|
14
|
+
class ExponentialBackoffRateLimiter
|
|
15
|
+
DEFAULT_BASE = 5.0
|
|
16
|
+
DEFAULT_MAX = 300.0
|
|
17
|
+
|
|
18
|
+
def initialize(base: DEFAULT_BASE, max: DEFAULT_MAX)
|
|
19
|
+
raise ArgumentError, "base must be positive" unless base.positive?
|
|
20
|
+
raise ArgumentError, "max must be positive" unless max.positive?
|
|
21
|
+
|
|
22
|
+
@base = base
|
|
23
|
+
@max = max
|
|
24
|
+
@failures = Hash.new(0)
|
|
25
|
+
@mutex = Mutex.new
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Records one failure for +key+ and returns the delay (seconds) the
|
|
29
|
+
# next attempt should be postponed by.
|
|
30
|
+
def backoff(key)
|
|
31
|
+
@mutex.synchronize do
|
|
32
|
+
@failures[key] += 1
|
|
33
|
+
[@max, @base * (2**(@failures[key] - 1))].min
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Clears the failure count for +key+ (call after a successful
|
|
38
|
+
# reconcile so the next failure starts from the base delay again).
|
|
39
|
+
def forget(key)
|
|
40
|
+
@mutex.synchronize { @failures.delete(key) }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Number of consecutive failures currently recorded for +key+.
|
|
44
|
+
def failures(key)
|
|
45
|
+
@mutex.synchronize { @failures[key] }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
attr_reader :base, :max
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|