kruby-controller 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +201 -0
  3. data/README.md +117 -0
  4. data/Rakefile +8 -0
  5. data/exe/kruby-controller +32 -0
  6. data/lib/kruby-controller.rb +8 -0
  7. data/lib/kruby_controller/builder/controller_builder.rb +52 -0
  8. data/lib/kruby_controller/cache/cache.rb +100 -0
  9. data/lib/kruby_controller/cache/deep_copy.rb +32 -0
  10. data/lib/kruby_controller/cache/delta.rb +12 -0
  11. data/lib/kruby_controller/cache/informer.rb +453 -0
  12. data/lib/kruby_controller/cache/request_error.rb +55 -0
  13. data/lib/kruby_controller/cache/store.rb +92 -0
  14. data/lib/kruby_controller/cache/watch_stream.rb +144 -0
  15. data/lib/kruby_controller/cli/create_api_command.rb +335 -0
  16. data/lib/kruby_controller/cli/flag_parser.rb +85 -0
  17. data/lib/kruby_controller/cli/init_command.rb +383 -0
  18. data/lib/kruby_controller/controller.rb +200 -0
  19. data/lib/kruby_controller/crd.rb +141 -0
  20. data/lib/kruby_controller/election/leader_elector.rb +212 -0
  21. data/lib/kruby_controller/finalizer/finalizer_manager.rb +31 -0
  22. data/lib/kruby_controller/generator/crd_generator.rb +109 -0
  23. data/lib/kruby_controller/gvk.rb +59 -0
  24. data/lib/kruby_controller/handler/enqueued_object.rb +18 -0
  25. data/lib/kruby_controller/handler/enqueued_owner.rb +51 -0
  26. data/lib/kruby_controller/handler/predicated.rb +21 -0
  27. data/lib/kruby_controller/kubernetes_object_client.rb +77 -0
  28. data/lib/kruby_controller/logger.rb +11 -0
  29. data/lib/kruby_controller/manager.rb +291 -0
  30. data/lib/kruby_controller/owner/owner_reference.rb +23 -0
  31. data/lib/kruby_controller/queue/rate_limiter.rb +51 -0
  32. data/lib/kruby_controller/queue/work_queue.rb +375 -0
  33. data/lib/kruby_controller/reconciler.rb +18 -0
  34. data/lib/kruby_controller/request.rb +59 -0
  35. data/lib/kruby_controller/result.rb +50 -0
  36. data/lib/kruby_controller/status/status_updater.rb +29 -0
  37. data/lib/kruby_controller/version.rb +5 -0
  38. data/lib/kruby_controller.rb +87 -0
  39. metadata +138 -0
@@ -0,0 +1,383 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "yaml"
5
+ require "kruby_controller/cli/flag_parser"
6
+
7
+ module KrubyController
8
+ module CLI
9
+ # `kruby-controller init` — scaffolds a new project (design doc §2.10.2).
10
+ class InitCommand
11
+ DEFAULT_DOMAIN = "example.com"
12
+ DEFAULT_LICENSE = "apache2"
13
+ SUPPORTED_LICENSES = %w[apache2 mit none].freeze
14
+ ALLOWED_FLAGS = %w[--domain --repo --license --force -h --help].freeze
15
+
16
+ USAGE = <<~USAGE.freeze
17
+ Usage: kruby-controller init [options]
18
+
19
+ Scaffolds a new KrubyController project.
20
+
21
+ Options:
22
+ --domain <domain> API domain suffix (default: #{DEFAULT_DOMAIN})
23
+ --repo <repo> Repository URL/path; the project directory is
24
+ created in the current directory under its last
25
+ segment (default: scaffold in the current
26
+ directory)
27
+ --license <license> #{SUPPORTED_LICENSES.join(' | ')} (default: #{DEFAULT_LICENSE})
28
+ --force Allow scaffolding into a non-empty directory
29
+ -h, --help Show this help
30
+ USAGE
31
+
32
+ def initialize(args)
33
+ @args = args
34
+ end
35
+
36
+ def run
37
+ if FlagParser.help?(@args)
38
+ puts USAGE
39
+ return 0
40
+ end
41
+
42
+ domain = FlagParser.flag(@args, "--domain") || DEFAULT_DOMAIN
43
+ repo = FlagParser.flag(@args, "--repo")
44
+ license = FlagParser.flag(@args, "--license") || DEFAULT_LICENSE
45
+ force = FlagParser.flag?(@args, "--force")
46
+ FlagParser.check_known!(@args, ALLOWED_FLAGS)
47
+
48
+ unless FlagParser.dns_subdomain?(domain)
49
+ raise UsageError,
50
+ "invalid --domain #{domain.inspect}; must be a DNS subdomain (lowercase labels separated by '.')"
51
+ end
52
+
53
+ unless SUPPORTED_LICENSES.include?(license)
54
+ raise UsageError, "unsupported license #{license.inspect}; expected one of #{SUPPORTED_LICENSES.join(', ')}"
55
+ end
56
+
57
+ @domain = domain
58
+ @repo = repo
59
+ @license = license
60
+ @name, @target_dir = resolve_target(repo)
61
+ validate_target!(force)
62
+
63
+ scaffold
64
+ 0
65
+ end
66
+
67
+ private
68
+
69
+ def resolve_target(repo)
70
+ if repo.nil? || repo == "."
71
+ name = FlagParser.normalize_dns_label(File.basename(Dir.pwd))
72
+ raise UsageError, "cannot derive a valid project name from #{Dir.pwd}" if name.nil?
73
+
74
+ return [name, Dir.pwd]
75
+ end
76
+
77
+ name = FlagParser.normalize_dns_label(File.basename(File.expand_path(repo)))
78
+ raise UsageError, "cannot derive a valid project name from --repo #{repo.inspect}" if name.nil?
79
+
80
+ [name, File.join(Dir.pwd, name)]
81
+ end
82
+
83
+ def validate_target!(force)
84
+ return unless File.exist?(@target_dir)
85
+
86
+ raise UsageError, "target path #{@target_dir} is a file" if File.file?(@target_dir)
87
+
88
+ entries = Dir.children(@target_dir)
89
+ return if entries.empty? || force
90
+
91
+ raise UsageError, "directory #{@target_dir} is not empty (use --force to scaffold anyway)"
92
+ end
93
+
94
+ def scaffold
95
+ puts "Initializing KrubyController project #{@name.inspect}..."
96
+ puts " domain: #{@domain}"
97
+ puts " license: #{@license}"
98
+
99
+ files = {
100
+ "Gemfile" => gemfile,
101
+ "Rakefile" => rakefile,
102
+ ".kruby-controller.yml" => project_yml,
103
+ "bin/manager" => bin_manager,
104
+ "cmd/manager.rb" => cmd_manager,
105
+ "config/crd/bases/.gitkeep" => "",
106
+ "config/rbac/role.yaml" => rbac_role,
107
+ "config/default.yaml" => default_deployment,
108
+ "pkg/crds/.gitkeep" => "",
109
+ "pkg/reconcilers/.gitkeep" => "",
110
+ "pkg/controller_suite_test.rb" => controller_suite_test,
111
+ "docs/README.md" => docs_readme,
112
+ ".gitignore" => gitignore
113
+ }
114
+ files["LICENSE"] = license_file if @license != "none"
115
+
116
+ files.each do |relative_path, content|
117
+ write_file(relative_path, content)
118
+ end
119
+ File.chmod(0o755, File.join(@target_dir, "bin/manager"))
120
+
121
+ puts "Created:"
122
+ files.each_key { |p| puts " #{display_path(p)}" }
123
+ puts "Done. Next steps:"
124
+ puts next_steps
125
+ 0
126
+ end
127
+
128
+ def write_file(relative_path, content)
129
+ path = File.join(@target_dir, relative_path)
130
+ FileUtils.mkdir_p(File.dirname(path))
131
+ File.write(path, content)
132
+ end
133
+
134
+ def display_path(relative_path)
135
+ @target_dir == Dir.pwd ? relative_path : File.join(@target_dir, relative_path)
136
+ end
137
+
138
+ def next_steps
139
+ lines = []
140
+ lines << " cd #{File.basename(@target_dir)}" unless @target_dir == Dir.pwd
141
+ lines << " bundle install"
142
+ lines << " kruby-controller create api --group stable --version v1 --kind CronTask"
143
+ lines.join("\n")
144
+ end
145
+
146
+ def gemfile
147
+ <<~RUBY
148
+ # frozen_string_literal: true
149
+
150
+ source "https://rubygems.org"
151
+
152
+ gem "kruby-controller"
153
+
154
+ group :development, :test do
155
+ gem "rake", "~> 13.0"
156
+ gem "rspec", "~> 3.13"
157
+ gem "rubocop", "~> 1.70"
158
+ end
159
+ RUBY
160
+ end
161
+
162
+ def rakefile
163
+ <<~RUBY
164
+ # frozen_string_literal: true
165
+
166
+ require "rspec/core/rake_task"
167
+
168
+ RSpec::Core::RakeTask.new(:spec) do |t|
169
+ t.pattern = "pkg/**/*_test.rb"
170
+ end
171
+
172
+ task default: :spec
173
+ RUBY
174
+ end
175
+
176
+ def project_yml
177
+ YAML.dump(
178
+ "name" => @name,
179
+ "domain" => @domain,
180
+ "repo" => @repo || ".",
181
+ "license" => @license
182
+ )
183
+ end
184
+
185
+ def bin_manager
186
+ <<~RUBY
187
+ #!/usr/bin/env ruby
188
+ # frozen_string_literal: true
189
+
190
+ require_relative "../cmd/manager"
191
+ RUBY
192
+ end
193
+
194
+ def cmd_manager
195
+ <<~RUBY
196
+ # frozen_string_literal: true
197
+
198
+ require "kruby-controller"
199
+
200
+ manager = KrubyController::Manager.new(
201
+ kube_config: ENV["KUBECONFIG"],
202
+ leader_elect: true
203
+ )
204
+
205
+ # Register controllers here. `kruby-controller create api` appends
206
+ # a manager.add(<Reconciler>, for: <Kind>) line before manager.start.
207
+
208
+ manager.start
209
+ RUBY
210
+ end
211
+
212
+ def rbac_role
213
+ <<~YAML
214
+ apiVersion: rbac.authorization.k8s.io/v1
215
+ kind: ClusterRole
216
+ metadata:
217
+ name: #{@name}-manager-role
218
+ rules:
219
+ # Leader election
220
+ - apiGroups: ["coordination.k8s.io"]
221
+ resources: ["leases"]
222
+ verbs: ["get", "create", "update", "patch"]
223
+ YAML
224
+ end
225
+
226
+ def default_deployment
227
+ <<~YAML
228
+ # Default manifests for the #{@name} manager.
229
+ apiVersion: v1
230
+ kind: Namespace
231
+ metadata:
232
+ name: #{@name}
233
+ ---
234
+ apiVersion: v1
235
+ kind: ServiceAccount
236
+ metadata:
237
+ name: #{@name}-manager
238
+ namespace: #{@name}
239
+ ---
240
+ # Bind the base manager ClusterRole (leader election) to the manager
241
+ # ServiceAccount so the deployment can acquire coordination leases.
242
+ apiVersion: rbac.authorization.k8s.io/v1
243
+ kind: ClusterRoleBinding
244
+ metadata:
245
+ name: #{@name}-manager-rolebinding
246
+ roleRef:
247
+ apiGroup: rbac.authorization.k8s.io
248
+ kind: ClusterRole
249
+ name: #{@name}-manager-role
250
+ subjects:
251
+ - kind: ServiceAccount
252
+ name: #{@name}-manager
253
+ namespace: #{@name}
254
+ ---
255
+ apiVersion: apps/v1
256
+ kind: Deployment
257
+ metadata:
258
+ name: #{@name}-manager
259
+ namespace: #{@name}
260
+ spec:
261
+ replicas: 1
262
+ selector:
263
+ matchLabels:
264
+ app: #{@name}
265
+ template:
266
+ metadata:
267
+ labels:
268
+ app: #{@name}
269
+ spec:
270
+ serviceAccountName: #{@name}-manager
271
+ containers:
272
+ - name: manager
273
+ image: #{@name}:latest # TODO: build and push your image
274
+ command: ["ruby", "bin/manager"]
275
+ YAML
276
+ end
277
+
278
+ def controller_suite_test
279
+ <<~RUBY
280
+ # frozen_string_literal: true
281
+
282
+ require "kruby-controller"
283
+
284
+ RSpec.describe "generated project" do
285
+ it "loads the kruby-controller framework" do
286
+ expect(KrubyController).to be_const_defined(:Manager)
287
+ end
288
+
289
+ it "can build a manager without a cluster" do
290
+ manager = KrubyController::Manager.new(kube_config: nil, leader_elect: false)
291
+ expect(manager).to be_a(KrubyController::Manager)
292
+ end
293
+ end
294
+ RUBY
295
+ end
296
+
297
+ def docs_readme
298
+ <<~MARKDOWN
299
+ # #{@name}
300
+
301
+ A Kubernetes controller project generated by `kruby-controller init`.
302
+
303
+ ## Quickstart
304
+
305
+ ```sh
306
+ bundle install
307
+ kruby-controller create api --group stable --version v1 --kind CronTask
308
+ bundle exec rake
309
+ ruby bin/manager
310
+ ```
311
+
312
+ ## Layout
313
+
314
+ | Path | Purpose |
315
+ |---|---|
316
+ | `cmd/manager.rb` | Manager entry point; `create api` appends `manager.add(...)` |
317
+ | `pkg/crds/` | CRD class definitions (KrubyController::CRD DSL) |
318
+ | `pkg/reconcilers/` | Reconciler implementations |
319
+ | `config/crd/bases/` | Generated CRD YAML manifests |
320
+ | `config/rbac/` | RBAC manifests |
321
+ | `config/default.yaml` | Namespace / ServiceAccount / Deployment |
322
+ MARKDOWN
323
+ end
324
+
325
+ def gitignore
326
+ <<~TEXT
327
+ .bundle/
328
+ vendor/bundle/
329
+ *.gem
330
+ TEXT
331
+ end
332
+
333
+ def license_file
334
+ case @license
335
+ when "apache2"
336
+ <<~LICENSE
337
+ Apache License
338
+ Version 2.0, January 2004
339
+ http://www.apache.org/licenses/
340
+
341
+ Copyright #{Time.now.year} #{@repo || @name}
342
+
343
+ Licensed under the Apache License, Version 2.0 (the "License");
344
+ you may not use this file except in compliance with the License.
345
+ You may obtain a copy of the License at
346
+
347
+ http://www.apache.org/licenses/LICENSE-2.0
348
+
349
+ Unless required by applicable law or agreed to in writing, software
350
+ distributed under the License is distributed on an "AS IS" BASIS,
351
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
352
+ implied. See the License for the specific language governing
353
+ permissions and limitations under the License.
354
+ LICENSE
355
+ when "mit"
356
+ <<~LICENSE
357
+ MIT License
358
+
359
+ Copyright (c) #{Time.now.year} #{@repo || @name}
360
+
361
+ Permission is hereby granted, free of charge, to any person obtaining a copy
362
+ of this software and associated documentation files (the "Software"), to deal
363
+ in the Software without restriction, including without limitation the rights
364
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
365
+ copies of the Software, and to permit persons to whom the Software is
366
+ furnished to do so, subject to the following conditions:
367
+
368
+ The above copyright notice and this permission notice shall be included in all
369
+ copies or substantial portions of the Software.
370
+
371
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
372
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
373
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
374
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
375
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
376
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
377
+ SOFTWARE.
378
+ LICENSE
379
+ end
380
+ end
381
+ end
382
+ end
383
+ end
@@ -0,0 +1,200 @@
1
+ # frozen_string_literal: true
2
+
3
+ module KrubyController
4
+ class Controller
5
+ attr_reader :name, :reconciler, :for_gvk, :rate_limiter
6
+
7
+ def initialize(reconciler:, cache:, for_gvk:, max_concurrent: 1, name: nil, owns: [], watches: [], predicate: nil,
8
+ rate_limiter: nil)
9
+ @reconciler = reconciler
10
+ @cache = cache
11
+ @for_gvk = for_gvk
12
+ @max_concurrent = max_concurrent
13
+ @name = name || @for_gvk.kind.downcase
14
+ @owns = Array(owns)
15
+ @watches = Array(watches)
16
+ @predicate = predicate
17
+ @rate_limiter = rate_limiter || Queue::ExponentialBackoffRateLimiter.new
18
+ @queue = Queue::WorkQueue.new(rate_limiter: @rate_limiter)
19
+ @running = false
20
+ @generation = 0
21
+ @reconciler.cache = cache
22
+ @reconciler.logger = KrubyController.logger
23
+ end
24
+
25
+ # Reopens the queue from a previous lifecycle BEFORE the registration
26
+ # guard: on leadership reacquisition the handlers are already registered
27
+ # (this method is a no-op) and the restarted cache emits its initial List
28
+ # diff while workers are still down, so the queue must accept events
29
+ # again or the changes observed while leaderless are silently dropped.
30
+ def register
31
+ @queue.reset if @queue.shutdown?
32
+ return if @handler_registered
33
+
34
+ informer = @cache.register(gvk: @for_gvk)
35
+ base = Handler::EnqueuedObject.new(queue: @queue)
36
+ informer.add_handler(@predicate ? Handler::Predicated.new(handler: base, predicate: @predicate) : base)
37
+
38
+ @owns.each do |owner_gvk|
39
+ owns_informer = @cache.register(gvk: owner_gvk)
40
+ owns_informer.add_handler(
41
+ Handler::EnqueuedOwner.new(queue: @queue, owner_gvk: @for_gvk, owner_namespaced: @for_gvk.namespaced?)
42
+ )
43
+ end
44
+
45
+ @watches.each do |watch|
46
+ watch_informer = @cache.register(gvk: watch[:gvk])
47
+ handler = watch[:handler] || Handler::EnqueuedObject.new(queue: @queue)
48
+ handler = Handler::Predicated.new(handler: handler, predicate: watch[:predicate]) if watch[:predicate]
49
+ watch_informer.add_handler(handler)
50
+ end
51
+
52
+ @handler_registered = true
53
+ end
54
+
55
+ def start
56
+ register
57
+ @running = true
58
+ # A generation token isolates workers across lifecycles: a worker
59
+ # abandoned by a timed-out stop (still inside a reconcile when stop
60
+ # gave up) must not resume pulling from the queue that a later start
61
+ # reopened for the NEW generation.
62
+ @generation += 1
63
+ generation = @generation
64
+ @workers = (1..@max_concurrent).map do |i|
65
+ Thread.new do
66
+ KrubyController.logger.info("Controller #{@name} worker #{i} started")
67
+ worker_loop(generation)
68
+ end
69
+ end
70
+ end
71
+
72
+ # Stops the controller. With +timeout+, waits at most `timeout` seconds
73
+ # in total for the workers to finish their in-flight reconcile; a worker
74
+ # that does not return within the remaining budget is abandoned AND
75
+ # removed from #workers so a later stop does not wait for it again (it
76
+ # exits on its next loop check, or on the generation check if the
77
+ # controller was restarted first). Without +timeout+ the join is
78
+ # unbounded (previous behavior).
79
+ def stop(timeout: nil)
80
+ @running = false
81
+ @queue.shutdown(@max_concurrent)
82
+ deadline = timeout && Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
83
+ abandoned = []
84
+ (@workers || []).each do |worker|
85
+ remaining = deadline && deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
86
+ if remaining && remaining <= 0
87
+ KrubyController.logger.warn(
88
+ "Controller #{@name}: stop budget exhausted; abandoning a running worker"
89
+ )
90
+ abandoned << worker
91
+ next
92
+ end
93
+
94
+ abandoned << worker unless join_worker(worker, remaining)
95
+ end
96
+ @workers -= abandoned unless abandoned.empty?
97
+ end
98
+
99
+ private
100
+
101
+ # Returns true when the worker exited within the timeout.
102
+ def join_worker(worker, timeout)
103
+ return worker.join if timeout.nil?
104
+ return true if worker.join(timeout)
105
+
106
+ KrubyController.logger.warn(
107
+ "Controller #{@name}: a reconcile did not return within #{timeout.round(1)}s; abandoning the worker"
108
+ )
109
+ false
110
+ end
111
+
112
+ def worker_loop(generation)
113
+ while @running && @generation == generation
114
+ request = @queue.get
115
+ break if request.equal?(Queue::WorkQueue::SENTINEL)
116
+
117
+ key = request.namespaced_name.to_s
118
+ failure = false
119
+ result = nil
120
+ begin
121
+ begin
122
+ result = @reconciler.reconcile(request)
123
+ rescue StandardError => e
124
+ failure = true
125
+ KrubyController.logger.error("Reconcile error for #{request}: #{e.message}")
126
+ end
127
+
128
+ if @queue.shutdown?
129
+ # Only a truly successful completion clears the failure history.
130
+ # Controller#start reopens the queue without replacing the
131
+ # limiter, so a stale streak would otherwise make the first
132
+ # failure after restart wait a doubled delay; but an error-bearing
133
+ # result (or a raised exception) is a failure and must keep its
134
+ # streak. The requeue itself is still suppressed.
135
+ @queue.forget(request) if clear_failure_on_shutdown?(failure, result)
136
+ return
137
+ end
138
+
139
+ # Rate-limit / forget bookkeeping runs while the key is still in
140
+ # flight, so a concurrent event for the same key is parked in the
141
+ # dirty set instead of racing the failure counter.
142
+ if failure
143
+ requeue_with_backoff(request, key)
144
+ else
145
+ handle_result(result, request, key)
146
+ end
147
+ ensure
148
+ @queue.done(request)
149
+ end
150
+ end
151
+ end
152
+
153
+ # Re-queues a failed reconcile after a per-key exponential backoff delay
154
+ # (5s base, doubling per consecutive failure, capped at 300s by default).
155
+ def requeue_with_backoff(request, key)
156
+ delay = @rate_limiter.backoff(key)
157
+ KrubyController.logger.info(
158
+ "Requeueing #{request} in #{delay} s " \
159
+ "(consecutive failures: #{@rate_limiter.failures(key)})"
160
+ )
161
+ @queue.add_after(request, delay)
162
+ end
163
+
164
+ def handle_result(result, request, key)
165
+ # A result carrying an error is a failure: rate-limited requeue,
166
+ # regardless of the explicit delay the reconciler requested. Checking
167
+ # this first keeps error-bearing RetryAfter responses on the exponential
168
+ # schedule instead of retrying on the fixed explicit delay.
169
+ if result.respond_to?(:error) && result.error
170
+ @queue.add_after(request, @rate_limiter.backoff(key))
171
+ return
172
+ end
173
+
174
+ # Error-free RetryAfter: honor the explicit delay and reset the streak.
175
+ if result.is_a?(Result::RetryAfter)
176
+ @queue.forget(request)
177
+ @queue.add_after(request, result.after)
178
+ return
179
+ end
180
+
181
+ # Ok, or an error-free Requeue: reset the streak. A Requeue additionally
182
+ # schedules an immediate retry.
183
+ @queue.forget(request)
184
+ @queue.add(request) if result.is_a?(Result::Requeue)
185
+ end
186
+
187
+ # Whether a completion that finishes while the queue is shutting down
188
+ # should clear its per-key failure history. Only a truly successful
189
+ # completion does: no raised exception (+failed+) and no error-bearing
190
+ # result. An error-bearing result (or exception) is a failure, and its
191
+ # streak must survive so the next failure after a restart does not drop
192
+ # back to the base delay.
193
+ def clear_failure_on_shutdown?(failed, result)
194
+ return false if failed
195
+ return false if result.respond_to?(:error) && !result.error.nil?
196
+
197
+ true
198
+ end
199
+ end
200
+ end