kubernetes-deploy 0.8.3 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 05fd6f9e6d7f1ed75f28c9bf9c25508be5a8ce84
4
- data.tar.gz: 8712e70aab07e4ce005feff75c4bad5441242508
3
+ metadata.gz: 7dea248a33790949829292f8e55958c44bacfc02
4
+ data.tar.gz: 95dd54cf01868dcd65f15a3dfe99c606e7913740
5
5
  SHA512:
6
- metadata.gz: c426af3a8c735dbf2c4543fb11ed84c7ae177a047790b4e53bf725124bd01ba1d77d0923da9123fb491bd6f1e982e7c99de287799b622e4bfc16e1a1e379fdc6
7
- data.tar.gz: 2f18a43c9929f2148ed9b614f46d7af115fec162235368301c9e3600bdd953c8f82c601bfa8e8d1687a8aa864746a103ddedecb4687df77b4a464404326733fa
6
+ metadata.gz: 50cf1550cc2fc26d303f0142ecf8f8515cad66eded851f319cbde3fee512f3039a238eebe7a9a2727637b541e268df8f3e81f740af26c06c05990ce1e6140a2d
7
+ data.tar.gz: e7a96a70120f591e67ea4cb6ed96b2933c09bf6f906969380a716c3b96ed16062f861fb09ffba453886778bcfc14dc69e8fc2c01ea0d2abc7295cb567c925a8b
@@ -124,7 +124,7 @@ module KubernetesDeploy
124
124
 
125
125
  events = fetch_events
126
126
  if events.present?
127
- helpful_info << " - Events:"
127
+ helpful_info << " - Events (common success events excluded):"
128
128
  events.each do |identifier, event_hashes|
129
129
  event_hashes.each { |event| helpful_info << " [#{identifier}]\t#{event}" }
130
130
  end
@@ -139,6 +139,11 @@ module KubernetesDeploy
139
139
  else
140
140
  sorted_logs = container_logs.sort_by { |_, log_lines| log_lines.length }
141
141
  sorted_logs.each do |identifier, log_lines|
142
+ if log_lines.empty?
143
+ helpful_info << " - Logs from container '#{identifier}': #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
144
+ next
145
+ end
146
+
142
147
  helpful_info << " - Logs from container '#{identifier}' (last #{LOG_LINE_COUNT} lines shown):"
143
148
  log_lines.each do |line|
144
149
  helpful_info << " #{line}"
@@ -209,7 +214,11 @@ module KubernetesDeploy
209
214
  %[(eq .involvedObject.kind "#{kind}")],
210
215
  %[(eq .involvedObject.name "#{name}")],
211
216
  '(ne .reason "Started")',
212
- '(ne .reason "Created")'
217
+ '(ne .reason "Created")',
218
+ '(ne .reason "SuccessfulCreate")',
219
+ '(ne .reason "Scheduled")',
220
+ '(ne .reason "Pulling")',
221
+ '(ne .reason "Pulled")'
213
222
  ]
214
223
  condition_start = "{{if and #{and_conditions.join(' ')}}}"
215
224
  field_part = FIELDS.map { |f| "{{#{f}}}" }.join(%({{print "#{FIELD_SEPARATOR}"}}))
@@ -45,6 +45,14 @@ module KubernetesDeploy
45
45
  @latest_rs && @latest_rs.deploy_failed?
46
46
  end
47
47
 
48
+ def failure_message
49
+ @latest_rs.failure_message
50
+ end
51
+
52
+ def timeout_message
53
+ @latest_rs.timeout_message
54
+ end
55
+
48
56
  def deploy_timed_out?
49
57
  super || @latest_rs && @latest_rs.deploy_timed_out?
50
58
  end
@@ -2,16 +2,16 @@
2
2
  module KubernetesDeploy
3
3
  class Pod < KubernetesResource
4
4
  TIMEOUT = 10.minutes
5
- SUSPICIOUS_CONTAINER_STATES = %w(ImagePullBackOff RunContainerError ErrImagePull CrashLoopBackOff).freeze
6
5
 
7
6
  def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started: nil)
8
7
  @parent = parent
9
8
  @deploy_started = deploy_started
10
- @containers = definition.fetch("spec", {}).fetch("containers", {}).map { |c| c["name"] }
9
+ @containers = definition.fetch("spec", {}).fetch("containers", []).map { |c| Container.new(c) }
11
10
  unless @containers.present?
12
11
  logger.summary.add_paragraph("Rendered template content:\n#{definition.to_yaml}")
13
12
  raise FatalDeploymentError, "Template is missing required field spec.containers"
14
13
  end
14
+ @containers += definition["spec"].fetch("initContainers", []).map { |c| Container.new(c, init_container: true) }
15
15
  super(namespace: namespace, context: context, definition: definition, logger: logger)
16
16
  end
17
17
 
@@ -23,15 +23,16 @@ module KubernetesDeploy
23
23
 
24
24
  if pod_data.present?
25
25
  @found = true
26
- interpret_pod_status_data(pod_data["status"], pod_data["metadata"]) # sets @phase, @status and @ready
27
- if @deploy_started
28
- log_suspicious_states(pod_data["status"].fetch("containerStatuses", []))
29
- end
26
+ @phase = @status = pod_data["status"]["phase"]
27
+ @status += " (Reason: #{pod_data['status']['reason']})" if pod_data['status']['reason'].present?
28
+ @ready = ready?(pod_data["status"])
29
+ update_container_statuses(pod_data["status"])
30
30
  else # reset
31
- @found = false
32
- @phase = @status = nil
33
- @ready = false
31
+ @found = @ready = false
32
+ @status = @phase = 'Unknown'
33
+ @containers.each(&:reset_status)
34
34
  end
35
+
35
36
  display_logs if unmanaged? && deploy_succeeded?
36
37
  end
37
38
 
@@ -44,13 +45,42 @@ module KubernetesDeploy
44
45
  end
45
46
 
46
47
  def deploy_failed?
47
- @phase == "Failed"
48
+ return true if @phase == "Failed"
49
+ @containers.any?(&:doomed?)
48
50
  end
49
51
 
50
52
  def exists?
51
53
  @found
52
54
  end
53
55
 
56
+ def timeout_message
57
+ return if unmanaged?
58
+ return unless @phase == "Running" && !@ready
59
+ pieces = ["Your pods are running, but the following containers seem to be failing their readiness probes:"]
60
+ @containers.each do |c|
61
+ next if c.init_container? || c.ready?
62
+ yellow_name = ColorizedString.new(c.name).yellow
63
+ pieces << "> #{yellow_name} must respond with a good status code at '#{c.probe_location}'"
64
+ end
65
+ pieces.join("\n") + "\n"
66
+ end
67
+
68
+ def failure_message
69
+ doomed_containers = @containers.select(&:doomed?)
70
+ return unless doomed_containers.present?
71
+ container_messages = doomed_containers.map do |c|
72
+ red_name = ColorizedString.new(c.name).red
73
+ "> #{red_name}: #{c.doom_reason}"
74
+ end
75
+
76
+ intro = if unmanaged?
77
+ "The following containers encountered errors:"
78
+ else
79
+ "The following containers are in a state that is unlikely to be recoverable:"
80
+ end
81
+ intro + "\n" + container_messages.join("\n") + "\n"
82
+ end
83
+
54
84
  # Returns a hash in the following format:
55
85
  # {
56
86
  # "app" => ["array of log lines", "received from app container"],
@@ -58,39 +88,35 @@ module KubernetesDeploy
58
88
  # }
59
89
  def fetch_logs
60
90
  return {} unless exists? && @containers.present?
61
- @containers.each_with_object({}) do |container_name, container_logs|
91
+ @containers.each_with_object({}) do |container, container_logs|
62
92
  cmd = [
63
93
  "logs",
64
94
  @name,
65
- "--container=#{container_name}",
95
+ "--container=#{container.name}",
66
96
  "--since-time=#{@deploy_started.to_datetime.rfc3339}",
67
97
  ]
68
98
  cmd << "--tail=#{LOG_LINE_COUNT}" unless unmanaged?
69
99
  out, _err, _st = kubectl.run(*cmd)
70
- container_logs[container_name] = out.split("\n")
100
+ container_logs[container.name] = out.split("\n")
71
101
  end
72
102
  end
73
103
 
74
104
  private
75
105
 
76
- def interpret_pod_status_data(status_data, metadata)
77
- @status = @phase = (metadata["deletionTimestamp"] ? "Terminating" : status_data["phase"])
78
-
79
- if @phase == "Failed" && status_data['reason'].present?
80
- @status += " (Reason: #{status_data['reason']})"
81
- elsif @phase != "Terminating"
82
- ready_condition = status_data.fetch("conditions", []).find { |condition| condition["type"] == "Ready" }
83
- @ready = ready_condition.present? && (ready_condition["status"] == "True")
84
- @status += " (Ready: #{@ready})"
85
- end
106
+ def ready?(status_data)
107
+ ready_condition = status_data.fetch("conditions", []).find { |condition| condition["type"] == "Ready" }
108
+ ready_condition.present? && (ready_condition["status"] == "True")
86
109
  end
87
110
 
88
- def log_suspicious_states(container_statuses)
89
- container_statuses.each do |status|
90
- waiting_state = status["state"]["waiting"] if status["state"]
91
- reason = waiting_state["reason"] if waiting_state
92
- next unless SUSPICIOUS_CONTAINER_STATES.include?(reason)
93
- @logger.warn("#{id} has container in state #{reason} (#{waiting_state['message']})")
111
+ def update_container_statuses(status_data)
112
+ @containers.each do |c|
113
+ key = c.init_container? ? "initContainerStatuses" : "containerStatuses"
114
+ if status_data.key?(key)
115
+ data = status_data[key].find { |st| st["name"] == c.name }
116
+ c.update_status(data)
117
+ else
118
+ c.reset_status
119
+ end
94
120
  end
95
121
  end
96
122
 
@@ -120,5 +146,69 @@ module KubernetesDeploy
120
146
 
121
147
  @already_displayed = true
122
148
  end
149
+
150
+ class Container
151
+ STATUS_SCAFFOLD = {
152
+ "state" => {
153
+ "running" => {},
154
+ "waiting" => {},
155
+ "terminated" => {},
156
+ },
157
+ "lastState" => {
158
+ "terminated" => {}
159
+ }
160
+ }.freeze
161
+
162
+ attr_reader :name, :probe_location
163
+
164
+ def initialize(definition, init_container: false)
165
+ @init_container = init_container
166
+ @name = definition["name"]
167
+ @image = definition["image"]
168
+ @probe_location = definition.fetch("readinessProbe", {}).fetch("httpGet", {})["path"]
169
+ @status = STATUS_SCAFFOLD.dup
170
+ end
171
+
172
+ def doomed?
173
+ doom_reason.present?
174
+ end
175
+
176
+ def doom_reason
177
+ exit_code = @status['lastState']['terminated']['exitCode']
178
+ last_terminated_reason = @status["lastState"]["terminated"]["reason"]
179
+ limbo_reason = @status["state"]["waiting"]["reason"]
180
+ limbo_message = @status["state"]["waiting"]["message"]
181
+
182
+ if last_terminated_reason == "ContainerCannotRun"
183
+ # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
184
+ "Failed to start (exit #{exit_code}): #{@status['lastState']['terminated']['message']}"
185
+ elsif limbo_reason == "CrashLoopBackOff"
186
+ "Crashing repeatedly (exit #{exit_code}). See logs for more information."
187
+ elsif %w(ImagePullBackOff ErrImagePull).include?(limbo_reason) && limbo_message.match("not found")
188
+ "Failed to pull image #{@image}. "\
189
+ "Did you wait for it to be built and pushed to the registry before deploying?"
190
+ elsif limbo_message == "Generate Container Config Failed"
191
+ # reason/message are backwards
192
+ # Flip this after https://github.com/kubernetes/kubernetes/commit/df41787b1a3f51b73fb6db8a2203f0a7c7c92931
193
+ "Failed to generate container configuration: #{limbo_reason}"
194
+ end
195
+ end
196
+
197
+ def ready?
198
+ @status['ready'] == "true"
199
+ end
200
+
201
+ def init_container?
202
+ @init_container
203
+ end
204
+
205
+ def update_status(data)
206
+ @status = STATUS_SCAFFOLD.deep_merge(data || {})
207
+ end
208
+
209
+ def reset_status
210
+ @status = STATUS_SCAFFOLD.dup
211
+ end
212
+ end
123
213
  end
124
214
  end
@@ -42,6 +42,14 @@ module KubernetesDeploy
42
42
  @pods.present? && @pods.all?(&:deploy_failed?)
43
43
  end
44
44
 
45
+ def failure_message
46
+ @pods.map(&:failure_message).compact.uniq.join("\n")
47
+ end
48
+
49
+ def timeout_message
50
+ @pods.map(&:timeout_message).compact.uniq.join("\n")
51
+ end
52
+
45
53
  def deploy_timed_out?
46
54
  super || @pods.present? && @pods.all?(&:deploy_timed_out?)
47
55
  end
@@ -53,7 +61,8 @@ module KubernetesDeploy
53
61
  def fetch_events
54
62
  own_events = super
55
63
  return own_events unless @pods.present?
56
- own_events.merge(@pods.first.fetch_events)
64
+ most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
65
+ own_events.merge(most_useful_pod.fetch_events)
57
66
  end
58
67
 
59
68
  def fetch_logs
@@ -76,7 +85,9 @@ module KubernetesDeploy
76
85
  end
77
86
 
78
87
  def container_names
79
- @definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
88
+ regular_containers = @definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
89
+ init_containers = @definition["spec"]["template"]["spec"].fetch("initContainers", {}).map { |c| c["name"] }
90
+ regular_containers + init_containers
80
91
  end
81
92
 
82
93
  def find_pods(rs_data)
@@ -6,21 +6,17 @@ module KubernetesDeploy
6
6
  def sync
7
7
  _, _err, st = kubectl.run("get", type, @name)
8
8
  @found = st.success?
9
- if @found
10
- endpoints, _err, st = kubectl.run("get", "endpoints", @name, "--output=jsonpath={.subsets[*].addresses[*].ip}")
11
- @num_endpoints = (st.success? ? endpoints.split.length : 0)
9
+ @related_deployment_replicas = fetch_related_replica_count
10
+ @status = if @num_pods_selected = fetch_related_pod_count
11
+ "Selects #{@num_pods_selected} #{'pod'.pluralize(@num_pods_selected)}"
12
12
  else
13
- @num_endpoints = 0
13
+ "Failed to count related pods"
14
14
  end
15
- @status = "#{@num_endpoints} endpoints"
16
15
  end
17
16
 
18
17
  def deploy_succeeded?
19
- if exposes_zero_replica_deployment?
20
- @num_endpoints == 0
21
- else
22
- @num_endpoints > 0
23
- end
18
+ # We can't use endpoints if we want the service to be able to fail fast when the pods are down
19
+ exposes_zero_replica_deployment? || selects_some_pods?
24
20
  end
25
21
 
26
22
  def deploy_failed?
@@ -28,10 +24,7 @@ module KubernetesDeploy
28
24
  end
29
25
 
30
26
  def timeout_message
31
- <<-MSG.strip_heredoc.strip
32
- This service does not have any endpoints. If the related pods are failing, fixing them will solve this as well.
33
- If the related pods are up, this service's selector is probably incorrect.
34
- MSG
27
+ "This service does not seem to select any pods. This means its spec.selector is probably incorrect."
35
28
  end
36
29
 
37
30
  def exists?
@@ -41,19 +34,32 @@ module KubernetesDeploy
41
34
  private
42
35
 
43
36
  def exposes_zero_replica_deployment?
44
- related_deployment_replicas && related_deployment_replicas == 0
37
+ return false unless @related_deployment_replicas
38
+ @related_deployment_replicas == 0
45
39
  end
46
40
 
47
- def related_deployment_replicas
48
- @related_deployment_replicas ||= begin
49
- selector = @definition["spec"]["selector"].map { |k, v| "#{k}=#{v}" }.join(",")
50
- raw_json, _err, st = kubectl.run("get", "deployments", "--selector=#{selector}", "--output=json")
51
- return unless st.success?
41
+ def selects_some_pods?
42
+ return false unless @num_pods_selected
43
+ @num_pods_selected > 0
44
+ end
52
45
 
53
- deployments = JSON.parse(raw_json)["items"]
54
- return unless deployments.length == 1
55
- deployments.first["spec"]["replicas"].to_i
56
- end
46
+ def selector
47
+ @selector ||= @definition["spec"]["selector"].map { |k, v| "#{k}=#{v}" }.join(",")
48
+ end
49
+
50
+ def fetch_related_pod_count
51
+ raw_json, _err, st = kubectl.run("get", "pods", "--selector=#{selector}", "--output=json")
52
+ return unless st.success?
53
+ JSON.parse(raw_json)["items"].length
54
+ end
55
+
56
+ def fetch_related_replica_count
57
+ raw_json, _err, st = kubectl.run("get", "deployments", "--selector=#{selector}", "--output=json")
58
+ return unless st.success?
59
+
60
+ deployments = JSON.parse(raw_json)["items"]
61
+ return unless deployments.length == 1
62
+ deployments.first["spec"]["replicas"].to_i
57
63
  end
58
64
  end
59
65
  end
@@ -157,6 +157,7 @@ module KubernetesDeploy
157
157
 
158
158
  if success_count > 0
159
159
  @logger.summary.add_action("successfully deployed #{success_count} #{'resource'.pluralize(success_count)}")
160
+ successful_resources.map(&:sync) # make sure we're printing the latest on resources that succeeded early
160
161
  final_statuses = successful_resources.map(&:pretty_status).join("\n")
161
162
  @logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
162
163
  end
@@ -1,3 +1,3 @@
1
1
  module KubernetesDeploy
2
- VERSION = "0.8.3"
2
+ VERSION = "0.9.0"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kubernetes-deploy
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.3
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Katrina Verey