kruby-controller 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +201 -0
- data/README.md +117 -0
- data/Rakefile +8 -0
- data/exe/kruby-controller +32 -0
- data/lib/kruby-controller.rb +8 -0
- data/lib/kruby_controller/builder/controller_builder.rb +52 -0
- data/lib/kruby_controller/cache/cache.rb +100 -0
- data/lib/kruby_controller/cache/deep_copy.rb +32 -0
- data/lib/kruby_controller/cache/delta.rb +12 -0
- data/lib/kruby_controller/cache/informer.rb +453 -0
- data/lib/kruby_controller/cache/request_error.rb +55 -0
- data/lib/kruby_controller/cache/store.rb +92 -0
- data/lib/kruby_controller/cache/watch_stream.rb +144 -0
- data/lib/kruby_controller/cli/create_api_command.rb +335 -0
- data/lib/kruby_controller/cli/flag_parser.rb +85 -0
- data/lib/kruby_controller/cli/init_command.rb +383 -0
- data/lib/kruby_controller/controller.rb +200 -0
- data/lib/kruby_controller/crd.rb +141 -0
- data/lib/kruby_controller/election/leader_elector.rb +212 -0
- data/lib/kruby_controller/finalizer/finalizer_manager.rb +31 -0
- data/lib/kruby_controller/generator/crd_generator.rb +109 -0
- data/lib/kruby_controller/gvk.rb +59 -0
- data/lib/kruby_controller/handler/enqueued_object.rb +18 -0
- data/lib/kruby_controller/handler/enqueued_owner.rb +51 -0
- data/lib/kruby_controller/handler/predicated.rb +21 -0
- data/lib/kruby_controller/kubernetes_object_client.rb +77 -0
- data/lib/kruby_controller/logger.rb +11 -0
- data/lib/kruby_controller/manager.rb +291 -0
- data/lib/kruby_controller/owner/owner_reference.rb +23 -0
- data/lib/kruby_controller/queue/rate_limiter.rb +51 -0
- data/lib/kruby_controller/queue/work_queue.rb +375 -0
- data/lib/kruby_controller/reconciler.rb +18 -0
- data/lib/kruby_controller/request.rb +59 -0
- data/lib/kruby_controller/result.rb +50 -0
- data/lib/kruby_controller/status/status_updater.rb +29 -0
- data/lib/kruby_controller/version.rb +5 -0
- data/lib/kruby_controller.rb +87 -0
- metadata +138 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "yaml"
|
|
5
|
+
require "kruby_controller/cli/flag_parser"
|
|
6
|
+
|
|
7
|
+
module KrubyController
|
|
8
|
+
module CLI
|
|
9
|
+
# `kruby-controller init` — scaffolds a new project (design doc §2.10.2).
|
|
10
|
+
class InitCommand
|
|
11
|
+
DEFAULT_DOMAIN = "example.com"
|
|
12
|
+
DEFAULT_LICENSE = "apache2"
|
|
13
|
+
SUPPORTED_LICENSES = %w[apache2 mit none].freeze
|
|
14
|
+
ALLOWED_FLAGS = %w[--domain --repo --license --force -h --help].freeze
|
|
15
|
+
|
|
16
|
+
USAGE = <<~USAGE.freeze
|
|
17
|
+
Usage: kruby-controller init [options]
|
|
18
|
+
|
|
19
|
+
Scaffolds a new KrubyController project.
|
|
20
|
+
|
|
21
|
+
Options:
|
|
22
|
+
--domain <domain> API domain suffix (default: #{DEFAULT_DOMAIN})
|
|
23
|
+
--repo <repo> Repository URL/path; the project directory is
|
|
24
|
+
created in the current directory under its last
|
|
25
|
+
segment (default: scaffold in the current
|
|
26
|
+
directory)
|
|
27
|
+
--license <license> #{SUPPORTED_LICENSES.join(' | ')} (default: #{DEFAULT_LICENSE})
|
|
28
|
+
--force Allow scaffolding into a non-empty directory
|
|
29
|
+
-h, --help Show this help
|
|
30
|
+
USAGE
|
|
31
|
+
|
|
32
|
+
def initialize(args)
|
|
33
|
+
@args = args
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def run
|
|
37
|
+
if FlagParser.help?(@args)
|
|
38
|
+
puts USAGE
|
|
39
|
+
return 0
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
domain = FlagParser.flag(@args, "--domain") || DEFAULT_DOMAIN
|
|
43
|
+
repo = FlagParser.flag(@args, "--repo")
|
|
44
|
+
license = FlagParser.flag(@args, "--license") || DEFAULT_LICENSE
|
|
45
|
+
force = FlagParser.flag?(@args, "--force")
|
|
46
|
+
FlagParser.check_known!(@args, ALLOWED_FLAGS)
|
|
47
|
+
|
|
48
|
+
unless FlagParser.dns_subdomain?(domain)
|
|
49
|
+
raise UsageError,
|
|
50
|
+
"invalid --domain #{domain.inspect}; must be a DNS subdomain (lowercase labels separated by '.')"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
unless SUPPORTED_LICENSES.include?(license)
|
|
54
|
+
raise UsageError, "unsupported license #{license.inspect}; expected one of #{SUPPORTED_LICENSES.join(', ')}"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
@domain = domain
|
|
58
|
+
@repo = repo
|
|
59
|
+
@license = license
|
|
60
|
+
@name, @target_dir = resolve_target(repo)
|
|
61
|
+
validate_target!(force)
|
|
62
|
+
|
|
63
|
+
scaffold
|
|
64
|
+
0
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def resolve_target(repo)
|
|
70
|
+
if repo.nil? || repo == "."
|
|
71
|
+
name = FlagParser.normalize_dns_label(File.basename(Dir.pwd))
|
|
72
|
+
raise UsageError, "cannot derive a valid project name from #{Dir.pwd}" if name.nil?
|
|
73
|
+
|
|
74
|
+
return [name, Dir.pwd]
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
name = FlagParser.normalize_dns_label(File.basename(File.expand_path(repo)))
|
|
78
|
+
raise UsageError, "cannot derive a valid project name from --repo #{repo.inspect}" if name.nil?
|
|
79
|
+
|
|
80
|
+
[name, File.join(Dir.pwd, name)]
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def validate_target!(force)
|
|
84
|
+
return unless File.exist?(@target_dir)
|
|
85
|
+
|
|
86
|
+
raise UsageError, "target path #{@target_dir} is a file" if File.file?(@target_dir)
|
|
87
|
+
|
|
88
|
+
entries = Dir.children(@target_dir)
|
|
89
|
+
return if entries.empty? || force
|
|
90
|
+
|
|
91
|
+
raise UsageError, "directory #{@target_dir} is not empty (use --force to scaffold anyway)"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def scaffold
|
|
95
|
+
puts "Initializing KrubyController project #{@name.inspect}..."
|
|
96
|
+
puts " domain: #{@domain}"
|
|
97
|
+
puts " license: #{@license}"
|
|
98
|
+
|
|
99
|
+
files = {
|
|
100
|
+
"Gemfile" => gemfile,
|
|
101
|
+
"Rakefile" => rakefile,
|
|
102
|
+
".kruby-controller.yml" => project_yml,
|
|
103
|
+
"bin/manager" => bin_manager,
|
|
104
|
+
"cmd/manager.rb" => cmd_manager,
|
|
105
|
+
"config/crd/bases/.gitkeep" => "",
|
|
106
|
+
"config/rbac/role.yaml" => rbac_role,
|
|
107
|
+
"config/default.yaml" => default_deployment,
|
|
108
|
+
"pkg/crds/.gitkeep" => "",
|
|
109
|
+
"pkg/reconcilers/.gitkeep" => "",
|
|
110
|
+
"pkg/controller_suite_test.rb" => controller_suite_test,
|
|
111
|
+
"docs/README.md" => docs_readme,
|
|
112
|
+
".gitignore" => gitignore
|
|
113
|
+
}
|
|
114
|
+
files["LICENSE"] = license_file if @license != "none"
|
|
115
|
+
|
|
116
|
+
files.each do |relative_path, content|
|
|
117
|
+
write_file(relative_path, content)
|
|
118
|
+
end
|
|
119
|
+
File.chmod(0o755, File.join(@target_dir, "bin/manager"))
|
|
120
|
+
|
|
121
|
+
puts "Created:"
|
|
122
|
+
files.each_key { |p| puts " #{display_path(p)}" }
|
|
123
|
+
puts "Done. Next steps:"
|
|
124
|
+
puts next_steps
|
|
125
|
+
0
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def write_file(relative_path, content)
|
|
129
|
+
path = File.join(@target_dir, relative_path)
|
|
130
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
131
|
+
File.write(path, content)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def display_path(relative_path)
|
|
135
|
+
@target_dir == Dir.pwd ? relative_path : File.join(@target_dir, relative_path)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def next_steps
|
|
139
|
+
lines = []
|
|
140
|
+
lines << " cd #{File.basename(@target_dir)}" unless @target_dir == Dir.pwd
|
|
141
|
+
lines << " bundle install"
|
|
142
|
+
lines << " kruby-controller create api --group stable --version v1 --kind CronTask"
|
|
143
|
+
lines.join("\n")
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def gemfile
|
|
147
|
+
<<~RUBY
|
|
148
|
+
# frozen_string_literal: true
|
|
149
|
+
|
|
150
|
+
source "https://rubygems.org"
|
|
151
|
+
|
|
152
|
+
gem "kruby-controller"
|
|
153
|
+
|
|
154
|
+
group :development, :test do
|
|
155
|
+
gem "rake", "~> 13.0"
|
|
156
|
+
gem "rspec", "~> 3.13"
|
|
157
|
+
gem "rubocop", "~> 1.70"
|
|
158
|
+
end
|
|
159
|
+
RUBY
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def rakefile
|
|
163
|
+
<<~RUBY
|
|
164
|
+
# frozen_string_literal: true
|
|
165
|
+
|
|
166
|
+
require "rspec/core/rake_task"
|
|
167
|
+
|
|
168
|
+
RSpec::Core::RakeTask.new(:spec) do |t|
|
|
169
|
+
t.pattern = "pkg/**/*_test.rb"
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
task default: :spec
|
|
173
|
+
RUBY
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def project_yml
|
|
177
|
+
YAML.dump(
|
|
178
|
+
"name" => @name,
|
|
179
|
+
"domain" => @domain,
|
|
180
|
+
"repo" => @repo || ".",
|
|
181
|
+
"license" => @license
|
|
182
|
+
)
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def bin_manager
|
|
186
|
+
<<~RUBY
|
|
187
|
+
#!/usr/bin/env ruby
|
|
188
|
+
# frozen_string_literal: true
|
|
189
|
+
|
|
190
|
+
require_relative "../cmd/manager"
|
|
191
|
+
RUBY
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def cmd_manager
|
|
195
|
+
<<~RUBY
|
|
196
|
+
# frozen_string_literal: true
|
|
197
|
+
|
|
198
|
+
require "kruby-controller"
|
|
199
|
+
|
|
200
|
+
manager = KrubyController::Manager.new(
|
|
201
|
+
kube_config: ENV["KUBECONFIG"],
|
|
202
|
+
leader_elect: true
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
# Register controllers here. `kruby-controller create api` appends
|
|
206
|
+
# a manager.add(<Reconciler>, for: <Kind>) line before manager.start.
|
|
207
|
+
|
|
208
|
+
manager.start
|
|
209
|
+
RUBY
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def rbac_role
|
|
213
|
+
<<~YAML
|
|
214
|
+
apiVersion: rbac.authorization.k8s.io/v1
|
|
215
|
+
kind: ClusterRole
|
|
216
|
+
metadata:
|
|
217
|
+
name: #{@name}-manager-role
|
|
218
|
+
rules:
|
|
219
|
+
# Leader election
|
|
220
|
+
- apiGroups: ["coordination.k8s.io"]
|
|
221
|
+
resources: ["leases"]
|
|
222
|
+
verbs: ["get", "create", "update", "patch"]
|
|
223
|
+
YAML
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def default_deployment
|
|
227
|
+
<<~YAML
|
|
228
|
+
# Default manifests for the #{@name} manager.
|
|
229
|
+
apiVersion: v1
|
|
230
|
+
kind: Namespace
|
|
231
|
+
metadata:
|
|
232
|
+
name: #{@name}
|
|
233
|
+
---
|
|
234
|
+
apiVersion: v1
|
|
235
|
+
kind: ServiceAccount
|
|
236
|
+
metadata:
|
|
237
|
+
name: #{@name}-manager
|
|
238
|
+
namespace: #{@name}
|
|
239
|
+
---
|
|
240
|
+
# Bind the base manager ClusterRole (leader election) to the manager
|
|
241
|
+
# ServiceAccount so the deployment can acquire coordination leases.
|
|
242
|
+
apiVersion: rbac.authorization.k8s.io/v1
|
|
243
|
+
kind: ClusterRoleBinding
|
|
244
|
+
metadata:
|
|
245
|
+
name: #{@name}-manager-rolebinding
|
|
246
|
+
roleRef:
|
|
247
|
+
apiGroup: rbac.authorization.k8s.io
|
|
248
|
+
kind: ClusterRole
|
|
249
|
+
name: #{@name}-manager-role
|
|
250
|
+
subjects:
|
|
251
|
+
- kind: ServiceAccount
|
|
252
|
+
name: #{@name}-manager
|
|
253
|
+
namespace: #{@name}
|
|
254
|
+
---
|
|
255
|
+
apiVersion: apps/v1
|
|
256
|
+
kind: Deployment
|
|
257
|
+
metadata:
|
|
258
|
+
name: #{@name}-manager
|
|
259
|
+
namespace: #{@name}
|
|
260
|
+
spec:
|
|
261
|
+
replicas: 1
|
|
262
|
+
selector:
|
|
263
|
+
matchLabels:
|
|
264
|
+
app: #{@name}
|
|
265
|
+
template:
|
|
266
|
+
metadata:
|
|
267
|
+
labels:
|
|
268
|
+
app: #{@name}
|
|
269
|
+
spec:
|
|
270
|
+
serviceAccountName: #{@name}-manager
|
|
271
|
+
containers:
|
|
272
|
+
- name: manager
|
|
273
|
+
image: #{@name}:latest # TODO: build and push your image
|
|
274
|
+
command: ["ruby", "bin/manager"]
|
|
275
|
+
YAML
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def controller_suite_test
|
|
279
|
+
<<~RUBY
|
|
280
|
+
# frozen_string_literal: true
|
|
281
|
+
|
|
282
|
+
require "kruby-controller"
|
|
283
|
+
|
|
284
|
+
RSpec.describe "generated project" do
|
|
285
|
+
it "loads the kruby-controller framework" do
|
|
286
|
+
expect(KrubyController).to be_const_defined(:Manager)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
it "can build a manager without a cluster" do
|
|
290
|
+
manager = KrubyController::Manager.new(kube_config: nil, leader_elect: false)
|
|
291
|
+
expect(manager).to be_a(KrubyController::Manager)
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
RUBY
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def docs_readme
|
|
298
|
+
<<~MARKDOWN
|
|
299
|
+
# #{@name}
|
|
300
|
+
|
|
301
|
+
A Kubernetes controller project generated by `kruby-controller init`.
|
|
302
|
+
|
|
303
|
+
## Quickstart
|
|
304
|
+
|
|
305
|
+
```sh
|
|
306
|
+
bundle install
|
|
307
|
+
kruby-controller create api --group stable --version v1 --kind CronTask
|
|
308
|
+
bundle exec rake
|
|
309
|
+
ruby bin/manager
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
## Layout
|
|
313
|
+
|
|
314
|
+
| Path | Purpose |
|
|
315
|
+
|---|---|
|
|
316
|
+
| `cmd/manager.rb` | Manager entry point; `create api` appends `manager.add(...)` |
|
|
317
|
+
| `pkg/crds/` | CRD class definitions (KrubyController::CRD DSL) |
|
|
318
|
+
| `pkg/reconcilers/` | Reconciler implementations |
|
|
319
|
+
| `config/crd/bases/` | Generated CRD YAML manifests |
|
|
320
|
+
| `config/rbac/` | RBAC manifests |
|
|
321
|
+
| `config/default.yaml` | Namespace / ServiceAccount / Deployment |
|
|
322
|
+
MARKDOWN
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def gitignore
|
|
326
|
+
<<~TEXT
|
|
327
|
+
.bundle/
|
|
328
|
+
vendor/bundle/
|
|
329
|
+
*.gem
|
|
330
|
+
TEXT
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def license_file
|
|
334
|
+
case @license
|
|
335
|
+
when "apache2"
|
|
336
|
+
<<~LICENSE
|
|
337
|
+
Apache License
|
|
338
|
+
Version 2.0, January 2004
|
|
339
|
+
http://www.apache.org/licenses/
|
|
340
|
+
|
|
341
|
+
Copyright #{Time.now.year} #{@repo || @name}
|
|
342
|
+
|
|
343
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
344
|
+
you may not use this file except in compliance with the License.
|
|
345
|
+
You may obtain a copy of the License at
|
|
346
|
+
|
|
347
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
348
|
+
|
|
349
|
+
Unless required by applicable law or agreed to in writing, software
|
|
350
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
351
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
352
|
+
implied. See the License for the specific language governing
|
|
353
|
+
permissions and limitations under the License.
|
|
354
|
+
LICENSE
|
|
355
|
+
when "mit"
|
|
356
|
+
<<~LICENSE
|
|
357
|
+
MIT License
|
|
358
|
+
|
|
359
|
+
Copyright (c) #{Time.now.year} #{@repo || @name}
|
|
360
|
+
|
|
361
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
362
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
363
|
+
in the Software without restriction, including without limitation the rights
|
|
364
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
365
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
366
|
+
furnished to do so, subject to the following conditions:
|
|
367
|
+
|
|
368
|
+
The above copyright notice and this permission notice shall be included in all
|
|
369
|
+
copies or substantial portions of the Software.
|
|
370
|
+
|
|
371
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
372
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
373
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
374
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
375
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
376
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
377
|
+
SOFTWARE.
|
|
378
|
+
LICENSE
|
|
379
|
+
end
|
|
380
|
+
end
|
|
381
|
+
end
|
|
382
|
+
end
|
|
383
|
+
end
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module KrubyController
|
|
4
|
+
class Controller
|
|
5
|
+
attr_reader :name, :reconciler, :for_gvk, :rate_limiter
|
|
6
|
+
|
|
7
|
+
def initialize(reconciler:, cache:, for_gvk:, max_concurrent: 1, name: nil, owns: [], watches: [], predicate: nil,
|
|
8
|
+
rate_limiter: nil)
|
|
9
|
+
@reconciler = reconciler
|
|
10
|
+
@cache = cache
|
|
11
|
+
@for_gvk = for_gvk
|
|
12
|
+
@max_concurrent = max_concurrent
|
|
13
|
+
@name = name || @for_gvk.kind.downcase
|
|
14
|
+
@owns = Array(owns)
|
|
15
|
+
@watches = Array(watches)
|
|
16
|
+
@predicate = predicate
|
|
17
|
+
@rate_limiter = rate_limiter || Queue::ExponentialBackoffRateLimiter.new
|
|
18
|
+
@queue = Queue::WorkQueue.new(rate_limiter: @rate_limiter)
|
|
19
|
+
@running = false
|
|
20
|
+
@generation = 0
|
|
21
|
+
@reconciler.cache = cache
|
|
22
|
+
@reconciler.logger = KrubyController.logger
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Reopens the queue from a previous lifecycle BEFORE the registration
|
|
26
|
+
# guard: on leadership reacquisition the handlers are already registered
|
|
27
|
+
# (this method is a no-op) and the restarted cache emits its initial List
|
|
28
|
+
# diff while workers are still down, so the queue must accept events
|
|
29
|
+
# again or the changes observed while leaderless are silently dropped.
|
|
30
|
+
def register
|
|
31
|
+
@queue.reset if @queue.shutdown?
|
|
32
|
+
return if @handler_registered
|
|
33
|
+
|
|
34
|
+
informer = @cache.register(gvk: @for_gvk)
|
|
35
|
+
base = Handler::EnqueuedObject.new(queue: @queue)
|
|
36
|
+
informer.add_handler(@predicate ? Handler::Predicated.new(handler: base, predicate: @predicate) : base)
|
|
37
|
+
|
|
38
|
+
@owns.each do |owner_gvk|
|
|
39
|
+
owns_informer = @cache.register(gvk: owner_gvk)
|
|
40
|
+
owns_informer.add_handler(
|
|
41
|
+
Handler::EnqueuedOwner.new(queue: @queue, owner_gvk: @for_gvk, owner_namespaced: @for_gvk.namespaced?)
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
@watches.each do |watch|
|
|
46
|
+
watch_informer = @cache.register(gvk: watch[:gvk])
|
|
47
|
+
handler = watch[:handler] || Handler::EnqueuedObject.new(queue: @queue)
|
|
48
|
+
handler = Handler::Predicated.new(handler: handler, predicate: watch[:predicate]) if watch[:predicate]
|
|
49
|
+
watch_informer.add_handler(handler)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
@handler_registered = true
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def start
|
|
56
|
+
register
|
|
57
|
+
@running = true
|
|
58
|
+
# A generation token isolates workers across lifecycles: a worker
|
|
59
|
+
# abandoned by a timed-out stop (still inside a reconcile when stop
|
|
60
|
+
# gave up) must not resume pulling from the queue that a later start
|
|
61
|
+
# reopened for the NEW generation.
|
|
62
|
+
@generation += 1
|
|
63
|
+
generation = @generation
|
|
64
|
+
@workers = (1..@max_concurrent).map do |i|
|
|
65
|
+
Thread.new do
|
|
66
|
+
KrubyController.logger.info("Controller #{@name} worker #{i} started")
|
|
67
|
+
worker_loop(generation)
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Stops the controller. With +timeout+, waits at most `timeout` seconds
|
|
73
|
+
# in total for the workers to finish their in-flight reconcile; a worker
|
|
74
|
+
# that does not return within the remaining budget is abandoned AND
|
|
75
|
+
# removed from #workers so a later stop does not wait for it again (it
|
|
76
|
+
# exits on its next loop check, or on the generation check if the
|
|
77
|
+
# controller was restarted first). Without +timeout+ the join is
|
|
78
|
+
# unbounded (previous behavior).
|
|
79
|
+
def stop(timeout: nil)
|
|
80
|
+
@running = false
|
|
81
|
+
@queue.shutdown(@max_concurrent)
|
|
82
|
+
deadline = timeout && Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
|
|
83
|
+
abandoned = []
|
|
84
|
+
(@workers || []).each do |worker|
|
|
85
|
+
remaining = deadline && deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
86
|
+
if remaining && remaining <= 0
|
|
87
|
+
KrubyController.logger.warn(
|
|
88
|
+
"Controller #{@name}: stop budget exhausted; abandoning a running worker"
|
|
89
|
+
)
|
|
90
|
+
abandoned << worker
|
|
91
|
+
next
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
abandoned << worker unless join_worker(worker, remaining)
|
|
95
|
+
end
|
|
96
|
+
@workers -= abandoned unless abandoned.empty?
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
# Returns true when the worker exited within the timeout.
|
|
102
|
+
def join_worker(worker, timeout)
|
|
103
|
+
return worker.join if timeout.nil?
|
|
104
|
+
return true if worker.join(timeout)
|
|
105
|
+
|
|
106
|
+
KrubyController.logger.warn(
|
|
107
|
+
"Controller #{@name}: a reconcile did not return within #{timeout.round(1)}s; abandoning the worker"
|
|
108
|
+
)
|
|
109
|
+
false
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def worker_loop(generation)
|
|
113
|
+
while @running && @generation == generation
|
|
114
|
+
request = @queue.get
|
|
115
|
+
break if request.equal?(Queue::WorkQueue::SENTINEL)
|
|
116
|
+
|
|
117
|
+
key = request.namespaced_name.to_s
|
|
118
|
+
failure = false
|
|
119
|
+
result = nil
|
|
120
|
+
begin
|
|
121
|
+
begin
|
|
122
|
+
result = @reconciler.reconcile(request)
|
|
123
|
+
rescue StandardError => e
|
|
124
|
+
failure = true
|
|
125
|
+
KrubyController.logger.error("Reconcile error for #{request}: #{e.message}")
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
if @queue.shutdown?
|
|
129
|
+
# Only a truly successful completion clears the failure history.
|
|
130
|
+
# Controller#start reopens the queue without replacing the
|
|
131
|
+
# limiter, so a stale streak would otherwise make the first
|
|
132
|
+
# failure after restart wait a doubled delay; but an error-bearing
|
|
133
|
+
# result (or a raised exception) is a failure and must keep its
|
|
134
|
+
# streak. The requeue itself is still suppressed.
|
|
135
|
+
@queue.forget(request) if clear_failure_on_shutdown?(failure, result)
|
|
136
|
+
return
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Rate-limit / forget bookkeeping runs while the key is still in
|
|
140
|
+
# flight, so a concurrent event for the same key is parked in the
|
|
141
|
+
# dirty set instead of racing the failure counter.
|
|
142
|
+
if failure
|
|
143
|
+
requeue_with_backoff(request, key)
|
|
144
|
+
else
|
|
145
|
+
handle_result(result, request, key)
|
|
146
|
+
end
|
|
147
|
+
ensure
|
|
148
|
+
@queue.done(request)
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Re-queues a failed reconcile after a per-key exponential backoff delay
|
|
154
|
+
# (5s base, doubling per consecutive failure, capped at 300s by default).
|
|
155
|
+
def requeue_with_backoff(request, key)
|
|
156
|
+
delay = @rate_limiter.backoff(key)
|
|
157
|
+
KrubyController.logger.info(
|
|
158
|
+
"Requeueing #{request} in #{delay} s " \
|
|
159
|
+
"(consecutive failures: #{@rate_limiter.failures(key)})"
|
|
160
|
+
)
|
|
161
|
+
@queue.add_after(request, delay)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def handle_result(result, request, key)
|
|
165
|
+
# A result carrying an error is a failure: rate-limited requeue,
|
|
166
|
+
# regardless of the explicit delay the reconciler requested. Checking
|
|
167
|
+
# this first keeps error-bearing RetryAfter responses on the exponential
|
|
168
|
+
# schedule instead of retrying on the fixed explicit delay.
|
|
169
|
+
if result.respond_to?(:error) && result.error
|
|
170
|
+
@queue.add_after(request, @rate_limiter.backoff(key))
|
|
171
|
+
return
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# Error-free RetryAfter: honor the explicit delay and reset the streak.
|
|
175
|
+
if result.is_a?(Result::RetryAfter)
|
|
176
|
+
@queue.forget(request)
|
|
177
|
+
@queue.add_after(request, result.after)
|
|
178
|
+
return
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Ok, or an error-free Requeue: reset the streak. A Requeue additionally
|
|
182
|
+
# schedules an immediate retry.
|
|
183
|
+
@queue.forget(request)
|
|
184
|
+
@queue.add(request) if result.is_a?(Result::Requeue)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Whether a completion that finishes while the queue is shutting down
|
|
188
|
+
# should clear its per-key failure history. Only a truly successful
|
|
189
|
+
# completion does: no raised exception (+failed+) and no error-bearing
|
|
190
|
+
# result. An error-bearing result (or exception) is a failure, and its
|
|
191
|
+
# streak must survive so the next failure after a restart does not drop
|
|
192
|
+
# back to the base delay.
|
|
193
|
+
def clear_failure_on_shutdown?(failed, result)
|
|
194
|
+
return false if failed
|
|
195
|
+
return false if result.respond_to?(:error) && !result.error.nil?
|
|
196
|
+
|
|
197
|
+
true
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
end
|