kubernetes-deploy 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +6 -0
- data/CHANGELOG.md +11 -1
- data/README.md +3 -3
- data/exe/kubernetes-deploy +11 -6
- data/exe/kubernetes-restart +7 -2
- data/lib/kubernetes-deploy/deferred_summary_logging.rb +8 -4
- data/lib/kubernetes-deploy/deploy_task.rb +49 -31
- data/lib/kubernetes-deploy/errors.rb +3 -0
- data/lib/kubernetes-deploy/resource_watcher.rb +12 -1
- data/lib/kubernetes-deploy/restart_task.rb +28 -9
- data/lib/kubernetes-deploy/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f0ae7839357ce28f77a544c170be1e439515205c
|
4
|
+
data.tar.gz: 945483435a69d88450f060d2a544cb046f93a17d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cc67d2dbc4ea9563f84bd11ae7c991144a0153223f21d07d14e6ad632396167d978c7a9a881386dc1a9a55756a5e06e4e9e60a1b1534b20b4705b4f38fb4d0a4
|
7
|
+
data.tar.gz: df05433e91bc432c9e7412c4806dbdfe4d5ba494763a0e4dfaf6ea6231ed1b1596c6e9dc4157e40f8dcf8857db828cbec9b098244964d147bcc84a2ce74c254e
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,7 +1,17 @@
|
|
1
|
+
### 0.17.1
|
2
|
+
*Features*
|
3
|
+
- kubernetes-restart and kubernetes-deploy use exit code 70 when a
|
4
|
+
deploy fails due to one or more resources failing to deploy in time.
|
5
|
+
([#244](https://github.com/Shopify/kubernetes-deploy/pull/244))
|
6
|
+
|
7
|
+
*Bug Fixes*
|
8
|
+
- Handle deploying thousands of resources at a time, previously kubernetes-deploy would fail with
|
9
|
+
`Argument list too long - kubectl (Errno::E2BIG)`. ([#257](https://github.com/Shopify/kubernetes-deploy/pull/257))
|
10
|
+
|
1
11
|
### 0.17.0
|
2
12
|
*Enhancements*
|
3
13
|
|
4
|
-
- Add the `--cascade` when we force replace a resource. ([#250](https://github.com/Shopify/kubernetes-deploy/pull/250))
|
14
|
+
- Add the `--cascade` flag when we force replace a resource. ([#250](https://github.com/Shopify/kubernetes-deploy/pull/250))
|
5
15
|
|
6
16
|
### 0.16.0
|
7
17
|
**Important:** This release changes the officially supported Kubernetes versions to v1.7 through v1.9. Other versions may continue to work, but we are no longer running our test suite against them.
|
data/README.md
CHANGED
@@ -70,9 +70,9 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
|
|
70
70
|
|
71
71
|
* Ruby 2.3+
|
72
72
|
* Your cluster must be running Kubernetes v1.7.0 or higher<sup>1</sup>
|
73
|
-
* Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#templates))
|
74
|
-
* You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory
|
75
|
-
* Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace
|
73
|
+
* Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#using-templates-and-variables))
|
74
|
+
* You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory.<sup>2</sup>
|
75
|
+
* Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.
|
76
76
|
|
77
77
|
<sup>1</sup> We run integration tests against these Kubernetes versions. Kubernetes v1.6 was officially supported in gem versions < 0.16. Kubernetes v1.5 was officially supported in gem versions < 0.12.
|
78
78
|
|
data/exe/kubernetes-deploy
CHANGED
@@ -63,9 +63,14 @@ runner = KubernetesDeploy::DeployTask.new(
|
|
63
63
|
logger: logger
|
64
64
|
)
|
65
65
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
66
|
+
begin
|
67
|
+
runner.run!(
|
68
|
+
verify_result: !skip_wait,
|
69
|
+
allow_protected_ns: allow_protected_ns,
|
70
|
+
prune: prune
|
71
|
+
)
|
72
|
+
rescue KubernetesDeploy::DeploymentTimeoutError
|
73
|
+
exit 70
|
74
|
+
rescue KubernetesDeploy::FatalDeploymentError
|
75
|
+
exit 1
|
76
|
+
end
|
data/exe/kubernetes-restart
CHANGED
@@ -17,5 +17,10 @@ context = ARGV[1]
|
|
17
17
|
logger = KubernetesDeploy::FormattedLogger.build(namespace, context)
|
18
18
|
|
19
19
|
restart = KubernetesDeploy::RestartTask.new(namespace: namespace, context: context, logger: logger)
|
20
|
-
|
21
|
-
|
20
|
+
begin
|
21
|
+
restart.perform!(raw_deployments)
|
22
|
+
rescue KubernetesDeploy::DeploymentTimeoutError
|
23
|
+
exit 70
|
24
|
+
rescue KubernetesDeploy::FatalDeploymentError
|
25
|
+
exit 1
|
26
|
+
end
|
@@ -34,12 +34,16 @@ module KubernetesDeploy
|
|
34
34
|
end
|
35
35
|
|
36
36
|
# Outputs the deferred summary information saved via @logger.summary.add_action and @logger.summary.add_paragraph
|
37
|
-
def print_summary(
|
38
|
-
|
39
|
-
|
37
|
+
def print_summary(status)
|
38
|
+
status_string = status.to_s.humanize.upcase
|
39
|
+
if status == :success
|
40
|
+
heading("Result: ", status_string, :green)
|
40
41
|
level = :info
|
42
|
+
elsif status == :timed_out
|
43
|
+
heading("Result: ", status_string, :yellow)
|
44
|
+
level = :fatal
|
41
45
|
else
|
42
|
-
heading("Result: ",
|
46
|
+
heading("Result: ", status_string, :red)
|
43
47
|
level = :fatal
|
44
48
|
end
|
45
49
|
|
@@ -3,6 +3,7 @@ require 'open3'
|
|
3
3
|
require 'yaml'
|
4
4
|
require 'shellwords'
|
5
5
|
require 'tempfile'
|
6
|
+
require 'fileutils'
|
6
7
|
require 'kubernetes-deploy/kubernetes_resource'
|
7
8
|
%w(
|
8
9
|
cloudsql
|
@@ -56,7 +57,6 @@ module KubernetesDeploy
|
|
56
57
|
kube-system
|
57
58
|
kube-public
|
58
59
|
)
|
59
|
-
|
60
60
|
# Things removed from default prune whitelist at https://github.com/kubernetes/kubernetes/blob/0dff56b4d88ec7551084bf89028dbeebf569620e/pkg/kubectl/cmd/apply.go#L411:
|
61
61
|
# core/v1/Namespace -- not namespaced
|
62
62
|
# core/v1/PersistentVolume -- not namespaced
|
@@ -105,7 +105,14 @@ module KubernetesDeploy
|
|
105
105
|
)
|
106
106
|
end
|
107
107
|
|
108
|
-
def run(
|
108
|
+
def run(*args)
|
109
|
+
run!(*args)
|
110
|
+
true
|
111
|
+
rescue FatalDeploymentError
|
112
|
+
false
|
113
|
+
end
|
114
|
+
|
115
|
+
def run!(verify_result: true, allow_protected_ns: false, prune: true)
|
109
116
|
start = Time.now.utc
|
110
117
|
@logger.reset
|
111
118
|
|
@@ -148,7 +155,12 @@ module KubernetesDeploy
|
|
148
155
|
start_normal_resource = Time.now.utc
|
149
156
|
deploy_resources(resources, prune: prune, verify: true)
|
150
157
|
::StatsD.measure('normal_resources.duration', StatsD.duration(start_normal_resource), tags: statsd_tags)
|
151
|
-
|
158
|
+
failed_resources = resources.reject(&:deploy_succeeded?)
|
159
|
+
success = failed_resources.empty?
|
160
|
+
if !success && failed_resources.all?(&:deploy_timed_out?)
|
161
|
+
raise DeploymentTimeoutError
|
162
|
+
end
|
163
|
+
raise FatalDeploymentError unless success
|
152
164
|
else
|
153
165
|
deploy_resources(resources, prune: prune, verify: false)
|
154
166
|
@logger.summary.add_action("deployed #{resources.length} #{'resource'.pluralize(resources.length)}")
|
@@ -157,16 +169,18 @@ module KubernetesDeploy
|
|
157
169
|
This means the desired changes were communicated to Kubernetes, but the deploy did not make sure they actually succeeded.
|
158
170
|
MSG
|
159
171
|
@logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
|
160
|
-
success = true
|
161
172
|
end
|
173
|
+
::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:success")
|
174
|
+
@logger.print_summary(:success)
|
175
|
+
rescue DeploymentTimeoutError
|
176
|
+
@logger.print_summary(:timed_out)
|
177
|
+
::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:timeout")
|
178
|
+
raise
|
162
179
|
rescue FatalDeploymentError => error
|
163
|
-
@logger.summary.add_action(error.message)
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
status = success ? "success" : "failed"
|
168
|
-
::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:#{status}")
|
169
|
-
success
|
180
|
+
@logger.summary.add_action(error.message) if error.message != error.class.to_s
|
181
|
+
@logger.print_summary(:failure)
|
182
|
+
::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:failed")
|
183
|
+
raise
|
170
184
|
end
|
171
185
|
|
172
186
|
private
|
@@ -366,29 +380,33 @@ module KubernetesDeploy
|
|
366
380
|
|
367
381
|
def apply_all(resources, prune)
|
368
382
|
return unless resources.present?
|
369
|
-
|
370
383
|
command = ["apply"]
|
371
|
-
resources.each do |r|
|
372
|
-
@logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
|
373
|
-
command.push("-f", r.file_path)
|
374
|
-
r.deploy_started_at = Time.now.utc
|
375
|
-
end
|
376
384
|
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
385
|
+
Dir.mktmpdir do |tmp_dir|
|
386
|
+
resources.each do |r|
|
387
|
+
@logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
|
388
|
+
FileUtils.symlink(r.file_path, tmp_dir)
|
389
|
+
r.deploy_started_at = Time.now.utc
|
390
|
+
end
|
391
|
+
command.push("-f", tmp_dir)
|
381
392
|
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
|
393
|
+
if prune
|
394
|
+
command.push("--prune", "--all")
|
395
|
+
prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
|
396
|
+
end
|
397
|
+
|
398
|
+
out, err, st = kubectl.run(*command, log_failure: false)
|
399
|
+
|
400
|
+
if st.success?
|
401
|
+
log_pruning(out) if prune
|
402
|
+
else
|
403
|
+
file_paths = find_bad_files_from_kubectl_output(err)
|
404
|
+
warn_msg = "WARNING: Any resources not mentioned in the error below were likely created/updated. " \
|
405
|
+
"You may wish to roll back this deploy."
|
406
|
+
@logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
|
407
|
+
record_invalid_template(err, file_paths: file_paths)
|
408
|
+
raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
|
409
|
+
end
|
392
410
|
end
|
393
411
|
end
|
394
412
|
|
@@ -81,7 +81,18 @@ module KubernetesDeploy
|
|
81
81
|
end
|
82
82
|
|
83
83
|
if fail_count > 0
|
84
|
-
|
84
|
+
timeouts, failures = failed_resources.partition(&:deploy_timed_out?)
|
85
|
+
if timeouts.present?
|
86
|
+
@logger.summary.add_action(
|
87
|
+
"timed out waiting for #{timeouts.length} #{'resource'.pluralize(timeouts.length)} to #{@operation_name}"
|
88
|
+
)
|
89
|
+
end
|
90
|
+
|
91
|
+
if failures.present?
|
92
|
+
@logger.summary.add_action(
|
93
|
+
"failed to #{@operation_name} #{failures.length} #{'resource'.pluralize(failures.length)}"
|
94
|
+
)
|
95
|
+
end
|
85
96
|
KubernetesDeploy::Concurrency.split_across_threads(failed_resources, &:sync_debug_info)
|
86
97
|
failed_resources.each { |r| @logger.summary.add_paragraph(r.debug_message) }
|
87
98
|
end
|
@@ -26,7 +26,14 @@ module KubernetesDeploy
|
|
26
26
|
@logger = logger
|
27
27
|
end
|
28
28
|
|
29
|
-
def perform(
|
29
|
+
def perform(*args)
|
30
|
+
perform!(*args)
|
31
|
+
true
|
32
|
+
rescue FatalDeploymentError
|
33
|
+
false
|
34
|
+
end
|
35
|
+
|
36
|
+
def perform!(deployments_names = nil)
|
30
37
|
start = Time.now.utc
|
31
38
|
@logger.reset
|
32
39
|
|
@@ -42,19 +49,31 @@ module KubernetesDeploy
|
|
42
49
|
@logger.phase_heading("Waiting for rollout")
|
43
50
|
resources = build_watchables(deployments, start)
|
44
51
|
ResourceWatcher.new(resources, logger: @logger, operation_name: "restart").run
|
45
|
-
|
52
|
+
failed_resources = resources.reject(&:deploy_succeeded?)
|
53
|
+
success = failed_resources.empty?
|
54
|
+
if !success && failed_resources.all?(&:deploy_timed_out?)
|
55
|
+
raise DeploymentTimeoutError
|
56
|
+
end
|
57
|
+
raise FatalDeploymentError unless success
|
58
|
+
::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
|
59
|
+
@logger.print_summary(:success)
|
60
|
+
rescue DeploymentTimeoutError
|
61
|
+
::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
|
62
|
+
@logger.print_summary(:timed_out)
|
63
|
+
raise
|
46
64
|
rescue FatalDeploymentError => error
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
status = success ? "success" : "failed"
|
52
|
-
tags = %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
|
53
|
-
::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags)
|
65
|
+
::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
|
66
|
+
@logger.summary.add_action(error.message) if error.message != error.class.to_s
|
67
|
+
@logger.print_summary(:failure)
|
68
|
+
raise
|
54
69
|
end
|
55
70
|
|
56
71
|
private
|
57
72
|
|
73
|
+
def tags(status, deployments)
|
74
|
+
%W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
|
75
|
+
end
|
76
|
+
|
58
77
|
def identify_target_deployments(deployment_names)
|
59
78
|
if deployment_names.nil?
|
60
79
|
@logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kubernetes-deploy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.18.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Katrina Verey
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-03
|
12
|
+
date: 2018-04-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|