kubernetes-deploy 0.8.3 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/kubernetes-deploy/kubernetes_resource.rb +11 -2
- data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb +8 -0
- data/lib/kubernetes-deploy/kubernetes_resource/pod.rb +119 -29
- data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb +13 -2
- data/lib/kubernetes-deploy/kubernetes_resource/service.rb +30 -24
- data/lib/kubernetes-deploy/runner.rb +1 -0
- data/lib/kubernetes-deploy/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7dea248a33790949829292f8e55958c44bacfc02
|
4
|
+
data.tar.gz: 95dd54cf01868dcd65f15a3dfe99c606e7913740
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 50cf1550cc2fc26d303f0142ecf8f8515cad66eded851f319cbde3fee512f3039a238eebe7a9a2727637b541e268df8f3e81f740af26c06c05990ce1e6140a2d
|
7
|
+
data.tar.gz: e7a96a70120f591e67ea4cb6ed96b2933c09bf6f906969380a716c3b96ed16062f861fb09ffba453886778bcfc14dc69e8fc2c01ea0d2abc7295cb567c925a8b
|
@@ -124,7 +124,7 @@ module KubernetesDeploy
|
|
124
124
|
|
125
125
|
events = fetch_events
|
126
126
|
if events.present?
|
127
|
-
helpful_info << " - Events:"
|
127
|
+
helpful_info << " - Events (common success events excluded):"
|
128
128
|
events.each do |identifier, event_hashes|
|
129
129
|
event_hashes.each { |event| helpful_info << " [#{identifier}]\t#{event}" }
|
130
130
|
end
|
@@ -139,6 +139,11 @@ module KubernetesDeploy
|
|
139
139
|
else
|
140
140
|
sorted_logs = container_logs.sort_by { |_, log_lines| log_lines.length }
|
141
141
|
sorted_logs.each do |identifier, log_lines|
|
142
|
+
if log_lines.empty?
|
143
|
+
helpful_info << " - Logs from container '#{identifier}': #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
|
144
|
+
next
|
145
|
+
end
|
146
|
+
|
142
147
|
helpful_info << " - Logs from container '#{identifier}' (last #{LOG_LINE_COUNT} lines shown):"
|
143
148
|
log_lines.each do |line|
|
144
149
|
helpful_info << " #{line}"
|
@@ -209,7 +214,11 @@ module KubernetesDeploy
|
|
209
214
|
%[(eq .involvedObject.kind "#{kind}")],
|
210
215
|
%[(eq .involvedObject.name "#{name}")],
|
211
216
|
'(ne .reason "Started")',
|
212
|
-
'(ne .reason "Created")'
|
217
|
+
'(ne .reason "Created")',
|
218
|
+
'(ne .reason "SuccessfulCreate")',
|
219
|
+
'(ne .reason "Scheduled")',
|
220
|
+
'(ne .reason "Pulling")',
|
221
|
+
'(ne .reason "Pulled")'
|
213
222
|
]
|
214
223
|
condition_start = "{{if and #{and_conditions.join(' ')}}}"
|
215
224
|
field_part = FIELDS.map { |f| "{{#{f}}}" }.join(%({{print "#{FIELD_SEPARATOR}"}}))
|
@@ -45,6 +45,14 @@ module KubernetesDeploy
|
|
45
45
|
@latest_rs && @latest_rs.deploy_failed?
|
46
46
|
end
|
47
47
|
|
48
|
+
def failure_message
|
49
|
+
@latest_rs.failure_message
|
50
|
+
end
|
51
|
+
|
52
|
+
def timeout_message
|
53
|
+
@latest_rs.timeout_message
|
54
|
+
end
|
55
|
+
|
48
56
|
def deploy_timed_out?
|
49
57
|
super || @latest_rs && @latest_rs.deploy_timed_out?
|
50
58
|
end
|
@@ -2,16 +2,16 @@
|
|
2
2
|
module KubernetesDeploy
|
3
3
|
class Pod < KubernetesResource
|
4
4
|
TIMEOUT = 10.minutes
|
5
|
-
SUSPICIOUS_CONTAINER_STATES = %w(ImagePullBackOff RunContainerError ErrImagePull CrashLoopBackOff).freeze
|
6
5
|
|
7
6
|
def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started: nil)
|
8
7
|
@parent = parent
|
9
8
|
@deploy_started = deploy_started
|
10
|
-
@containers = definition.fetch("spec", {}).fetch("containers",
|
9
|
+
@containers = definition.fetch("spec", {}).fetch("containers", []).map { |c| Container.new(c) }
|
11
10
|
unless @containers.present?
|
12
11
|
logger.summary.add_paragraph("Rendered template content:\n#{definition.to_yaml}")
|
13
12
|
raise FatalDeploymentError, "Template is missing required field spec.containers"
|
14
13
|
end
|
14
|
+
@containers += definition["spec"].fetch("initContainers", []).map { |c| Container.new(c, init_container: true) }
|
15
15
|
super(namespace: namespace, context: context, definition: definition, logger: logger)
|
16
16
|
end
|
17
17
|
|
@@ -23,15 +23,16 @@ module KubernetesDeploy
|
|
23
23
|
|
24
24
|
if pod_data.present?
|
25
25
|
@found = true
|
26
|
-
|
27
|
-
if
|
28
|
-
|
29
|
-
|
26
|
+
@phase = @status = pod_data["status"]["phase"]
|
27
|
+
@status += " (Reason: #{pod_data['status']['reason']})" if pod_data['status']['reason'].present?
|
28
|
+
@ready = ready?(pod_data["status"])
|
29
|
+
update_container_statuses(pod_data["status"])
|
30
30
|
else # reset
|
31
|
-
@found = false
|
32
|
-
@
|
33
|
-
@
|
31
|
+
@found = @ready = false
|
32
|
+
@status = @phase = 'Unknown'
|
33
|
+
@containers.each(&:reset_status)
|
34
34
|
end
|
35
|
+
|
35
36
|
display_logs if unmanaged? && deploy_succeeded?
|
36
37
|
end
|
37
38
|
|
@@ -44,13 +45,42 @@ module KubernetesDeploy
|
|
44
45
|
end
|
45
46
|
|
46
47
|
def deploy_failed?
|
47
|
-
@phase == "Failed"
|
48
|
+
return true if @phase == "Failed"
|
49
|
+
@containers.any?(&:doomed?)
|
48
50
|
end
|
49
51
|
|
50
52
|
def exists?
|
51
53
|
@found
|
52
54
|
end
|
53
55
|
|
56
|
+
def timeout_message
|
57
|
+
return if unmanaged?
|
58
|
+
return unless @phase == "Running" && !@ready
|
59
|
+
pieces = ["Your pods are running, but the following containers seem to be failing their readiness probes:"]
|
60
|
+
@containers.each do |c|
|
61
|
+
next if c.init_container? || c.ready?
|
62
|
+
yellow_name = ColorizedString.new(c.name).yellow
|
63
|
+
pieces << "> #{yellow_name} must respond with a good status code at '#{c.probe_location}'"
|
64
|
+
end
|
65
|
+
pieces.join("\n") + "\n"
|
66
|
+
end
|
67
|
+
|
68
|
+
def failure_message
|
69
|
+
doomed_containers = @containers.select(&:doomed?)
|
70
|
+
return unless doomed_containers.present?
|
71
|
+
container_messages = doomed_containers.map do |c|
|
72
|
+
red_name = ColorizedString.new(c.name).red
|
73
|
+
"> #{red_name}: #{c.doom_reason}"
|
74
|
+
end
|
75
|
+
|
76
|
+
intro = if unmanaged?
|
77
|
+
"The following containers encountered errors:"
|
78
|
+
else
|
79
|
+
"The following containers are in a state that is unlikely to be recoverable:"
|
80
|
+
end
|
81
|
+
intro + "\n" + container_messages.join("\n") + "\n"
|
82
|
+
end
|
83
|
+
|
54
84
|
# Returns a hash in the following format:
|
55
85
|
# {
|
56
86
|
# "app" => ["array of log lines", "received from app container"],
|
@@ -58,39 +88,35 @@ module KubernetesDeploy
|
|
58
88
|
# }
|
59
89
|
def fetch_logs
|
60
90
|
return {} unless exists? && @containers.present?
|
61
|
-
@containers.each_with_object({}) do |
|
91
|
+
@containers.each_with_object({}) do |container, container_logs|
|
62
92
|
cmd = [
|
63
93
|
"logs",
|
64
94
|
@name,
|
65
|
-
"--container=#{
|
95
|
+
"--container=#{container.name}",
|
66
96
|
"--since-time=#{@deploy_started.to_datetime.rfc3339}",
|
67
97
|
]
|
68
98
|
cmd << "--tail=#{LOG_LINE_COUNT}" unless unmanaged?
|
69
99
|
out, _err, _st = kubectl.run(*cmd)
|
70
|
-
container_logs[
|
100
|
+
container_logs[container.name] = out.split("\n")
|
71
101
|
end
|
72
102
|
end
|
73
103
|
|
74
104
|
private
|
75
105
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
if @phase == "Failed" && status_data['reason'].present?
|
80
|
-
@status += " (Reason: #{status_data['reason']})"
|
81
|
-
elsif @phase != "Terminating"
|
82
|
-
ready_condition = status_data.fetch("conditions", []).find { |condition| condition["type"] == "Ready" }
|
83
|
-
@ready = ready_condition.present? && (ready_condition["status"] == "True")
|
84
|
-
@status += " (Ready: #{@ready})"
|
85
|
-
end
|
106
|
+
def ready?(status_data)
|
107
|
+
ready_condition = status_data.fetch("conditions", []).find { |condition| condition["type"] == "Ready" }
|
108
|
+
ready_condition.present? && (ready_condition["status"] == "True")
|
86
109
|
end
|
87
110
|
|
88
|
-
def
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
111
|
+
def update_container_statuses(status_data)
|
112
|
+
@containers.each do |c|
|
113
|
+
key = c.init_container? ? "initContainerStatuses" : "containerStatuses"
|
114
|
+
if status_data.key?(key)
|
115
|
+
data = status_data[key].find { |st| st["name"] == c.name }
|
116
|
+
c.update_status(data)
|
117
|
+
else
|
118
|
+
c.reset_status
|
119
|
+
end
|
94
120
|
end
|
95
121
|
end
|
96
122
|
|
@@ -120,5 +146,69 @@ module KubernetesDeploy
|
|
120
146
|
|
121
147
|
@already_displayed = true
|
122
148
|
end
|
149
|
+
|
150
|
+
class Container
|
151
|
+
STATUS_SCAFFOLD = {
|
152
|
+
"state" => {
|
153
|
+
"running" => {},
|
154
|
+
"waiting" => {},
|
155
|
+
"terminated" => {},
|
156
|
+
},
|
157
|
+
"lastState" => {
|
158
|
+
"terminated" => {}
|
159
|
+
}
|
160
|
+
}.freeze
|
161
|
+
|
162
|
+
attr_reader :name, :probe_location
|
163
|
+
|
164
|
+
def initialize(definition, init_container: false)
|
165
|
+
@init_container = init_container
|
166
|
+
@name = definition["name"]
|
167
|
+
@image = definition["image"]
|
168
|
+
@probe_location = definition.fetch("readinessProbe", {}).fetch("httpGet", {})["path"]
|
169
|
+
@status = STATUS_SCAFFOLD.dup
|
170
|
+
end
|
171
|
+
|
172
|
+
def doomed?
|
173
|
+
doom_reason.present?
|
174
|
+
end
|
175
|
+
|
176
|
+
def doom_reason
|
177
|
+
exit_code = @status['lastState']['terminated']['exitCode']
|
178
|
+
last_terminated_reason = @status["lastState"]["terminated"]["reason"]
|
179
|
+
limbo_reason = @status["state"]["waiting"]["reason"]
|
180
|
+
limbo_message = @status["state"]["waiting"]["message"]
|
181
|
+
|
182
|
+
if last_terminated_reason == "ContainerCannotRun"
|
183
|
+
# ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
|
184
|
+
"Failed to start (exit #{exit_code}): #{@status['lastState']['terminated']['message']}"
|
185
|
+
elsif limbo_reason == "CrashLoopBackOff"
|
186
|
+
"Crashing repeatedly (exit #{exit_code}). See logs for more information."
|
187
|
+
elsif %w(ImagePullBackOff ErrImagePull).include?(limbo_reason) && limbo_message.match("not found")
|
188
|
+
"Failed to pull image #{@image}. "\
|
189
|
+
"Did you wait for it to be built and pushed to the registry before deploying?"
|
190
|
+
elsif limbo_message == "Generate Container Config Failed"
|
191
|
+
# reason/message are backwards
|
192
|
+
# Flip this after https://github.com/kubernetes/kubernetes/commit/df41787b1a3f51b73fb6db8a2203f0a7c7c92931
|
193
|
+
"Failed to generate container configuration: #{limbo_reason}"
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
def ready?
|
198
|
+
@status['ready'] == "true"
|
199
|
+
end
|
200
|
+
|
201
|
+
def init_container?
|
202
|
+
@init_container
|
203
|
+
end
|
204
|
+
|
205
|
+
def update_status(data)
|
206
|
+
@status = STATUS_SCAFFOLD.deep_merge(data || {})
|
207
|
+
end
|
208
|
+
|
209
|
+
def reset_status
|
210
|
+
@status = STATUS_SCAFFOLD.dup
|
211
|
+
end
|
212
|
+
end
|
123
213
|
end
|
124
214
|
end
|
@@ -42,6 +42,14 @@ module KubernetesDeploy
|
|
42
42
|
@pods.present? && @pods.all?(&:deploy_failed?)
|
43
43
|
end
|
44
44
|
|
45
|
+
def failure_message
|
46
|
+
@pods.map(&:failure_message).compact.uniq.join("\n")
|
47
|
+
end
|
48
|
+
|
49
|
+
def timeout_message
|
50
|
+
@pods.map(&:timeout_message).compact.uniq.join("\n")
|
51
|
+
end
|
52
|
+
|
45
53
|
def deploy_timed_out?
|
46
54
|
super || @pods.present? && @pods.all?(&:deploy_timed_out?)
|
47
55
|
end
|
@@ -53,7 +61,8 @@ module KubernetesDeploy
|
|
53
61
|
def fetch_events
|
54
62
|
own_events = super
|
55
63
|
return own_events unless @pods.present?
|
56
|
-
|
64
|
+
most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
|
65
|
+
own_events.merge(most_useful_pod.fetch_events)
|
57
66
|
end
|
58
67
|
|
59
68
|
def fetch_logs
|
@@ -76,7 +85,9 @@ module KubernetesDeploy
|
|
76
85
|
end
|
77
86
|
|
78
87
|
def container_names
|
79
|
-
@definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
|
88
|
+
regular_containers = @definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
|
89
|
+
init_containers = @definition["spec"]["template"]["spec"].fetch("initContainers", {}).map { |c| c["name"] }
|
90
|
+
regular_containers + init_containers
|
80
91
|
end
|
81
92
|
|
82
93
|
def find_pods(rs_data)
|
@@ -6,21 +6,17 @@ module KubernetesDeploy
|
|
6
6
|
def sync
|
7
7
|
_, _err, st = kubectl.run("get", type, @name)
|
8
8
|
@found = st.success?
|
9
|
-
|
10
|
-
|
11
|
-
@
|
9
|
+
@related_deployment_replicas = fetch_related_replica_count
|
10
|
+
@status = if @num_pods_selected = fetch_related_pod_count
|
11
|
+
"Selects #{@num_pods_selected} #{'pod'.pluralize(@num_pods_selected)}"
|
12
12
|
else
|
13
|
-
|
13
|
+
"Failed to count related pods"
|
14
14
|
end
|
15
|
-
@status = "#{@num_endpoints} endpoints"
|
16
15
|
end
|
17
16
|
|
18
17
|
def deploy_succeeded?
|
19
|
-
if
|
20
|
-
|
21
|
-
else
|
22
|
-
@num_endpoints > 0
|
23
|
-
end
|
18
|
+
# We can't use endpoints if we want the service to be able to fail fast when the pods are down
|
19
|
+
exposes_zero_replica_deployment? || selects_some_pods?
|
24
20
|
end
|
25
21
|
|
26
22
|
def deploy_failed?
|
@@ -28,10 +24,7 @@ module KubernetesDeploy
|
|
28
24
|
end
|
29
25
|
|
30
26
|
def timeout_message
|
31
|
-
|
32
|
-
This service does not have any endpoints. If the related pods are failing, fixing them will solve this as well.
|
33
|
-
If the related pods are up, this service's selector is probably incorrect.
|
34
|
-
MSG
|
27
|
+
"This service does not seem to select any pods. This means its spec.selector is probably incorrect."
|
35
28
|
end
|
36
29
|
|
37
30
|
def exists?
|
@@ -41,19 +34,32 @@ module KubernetesDeploy
|
|
41
34
|
private
|
42
35
|
|
43
36
|
def exposes_zero_replica_deployment?
|
44
|
-
|
37
|
+
return false unless @related_deployment_replicas
|
38
|
+
@related_deployment_replicas == 0
|
45
39
|
end
|
46
40
|
|
47
|
-
def
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
return unless st.success?
|
41
|
+
def selects_some_pods?
|
42
|
+
return false unless @num_pods_selected
|
43
|
+
@num_pods_selected > 0
|
44
|
+
end
|
52
45
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
46
|
+
def selector
|
47
|
+
@selector ||= @definition["spec"]["selector"].map { |k, v| "#{k}=#{v}" }.join(",")
|
48
|
+
end
|
49
|
+
|
50
|
+
def fetch_related_pod_count
|
51
|
+
raw_json, _err, st = kubectl.run("get", "pods", "--selector=#{selector}", "--output=json")
|
52
|
+
return unless st.success?
|
53
|
+
JSON.parse(raw_json)["items"].length
|
54
|
+
end
|
55
|
+
|
56
|
+
def fetch_related_replica_count
|
57
|
+
raw_json, _err, st = kubectl.run("get", "deployments", "--selector=#{selector}", "--output=json")
|
58
|
+
return unless st.success?
|
59
|
+
|
60
|
+
deployments = JSON.parse(raw_json)["items"]
|
61
|
+
return unless deployments.length == 1
|
62
|
+
deployments.first["spec"]["replicas"].to_i
|
57
63
|
end
|
58
64
|
end
|
59
65
|
end
|
@@ -157,6 +157,7 @@ module KubernetesDeploy
|
|
157
157
|
|
158
158
|
if success_count > 0
|
159
159
|
@logger.summary.add_action("successfully deployed #{success_count} #{'resource'.pluralize(success_count)}")
|
160
|
+
successful_resources.map(&:sync) # make sure we're printing the latest on resources that succeeded early
|
160
161
|
final_statuses = successful_resources.map(&:pretty_status).join("\n")
|
161
162
|
@logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
|
162
163
|
end
|