kubernetes-deploy 0.8.3 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05fd6f9e6d7f1ed75f28c9bf9c25508be5a8ce84
-  data.tar.gz: 8712e70aab07e4ce005feff75c4bad5441242508
+  metadata.gz: 7dea248a33790949829292f8e55958c44bacfc02
+  data.tar.gz: 95dd54cf01868dcd65f15a3dfe99c606e7913740
 SHA512:
-  metadata.gz: c426af3a8c735dbf2c4543fb11ed84c7ae177a047790b4e53bf725124bd01ba1d77d0923da9123fb491bd6f1e982e7c99de287799b622e4bfc16e1a1e379fdc6
-  data.tar.gz: 2f18a43c9929f2148ed9b614f46d7af115fec162235368301c9e3600bdd953c8f82c601bfa8e8d1687a8aa864746a103ddedecb4687df77b4a464404326733fa
+  metadata.gz: 50cf1550cc2fc26d303f0142ecf8f8515cad66eded851f319cbde3fee512f3039a238eebe7a9a2727637b541e268df8f3e81f740af26c06c05990ce1e6140a2d
+  data.tar.gz: e7a96a70120f591e67ea4cb6ed96b2933c09bf6f906969380a716c3b96ed16062f861fb09ffba453886778bcfc14dc69e8fc2c01ea0d2abc7295cb567c925a8b

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -124,7 +124,7 @@ module KubernetesDeploy
 
       events = fetch_events
       if events.present?
-        helpful_info << "  - Events:"
+        helpful_info << "  - Events (common success events excluded):"
         events.each do |identifier, event_hashes|
           event_hashes.each { |event| helpful_info << "      [#{identifier}]\t#{event}" }
         end
@@ -139,6 +139,11 @@ module KubernetesDeploy
         else
           sorted_logs = container_logs.sort_by { |_, log_lines| log_lines.length }
           sorted_logs.each do |identifier, log_lines|
+            if log_lines.empty?
+              helpful_info << "  - Logs from container '#{identifier}': #{DEBUG_RESOURCE_NOT_FOUND_MESSAGE}"
+              next
+            end
+
             helpful_info << "  - Logs from container '#{identifier}' (last #{LOG_LINE_COUNT} lines shown):"
             log_lines.each do |line|
               helpful_info << "      #{line}"
@@ -209,7 +214,11 @@ module KubernetesDeploy
           %[(eq .involvedObject.kind "#{kind}")],
           %[(eq .involvedObject.name "#{name}")],
           '(ne .reason "Started")',
-          '(ne .reason "Created")'
+          '(ne .reason "Created")',
+          '(ne .reason "SuccessfulCreate")',
+          '(ne .reason "Scheduled")',
+          '(ne .reason "Pulling")',
+          '(ne .reason "Pulled")'
         ]
         condition_start = "{{if and #{and_conditions.join(' ')}}}"
         field_part = FIELDS.map { |f| "{{#{f}}}" }.join(%({{print "#{FIELD_SEPARATOR}"}}))

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -45,6 +45,14 @@ module KubernetesDeploy
       @latest_rs && @latest_rs.deploy_failed?
     end
 
+    def failure_message
+      @latest_rs.failure_message
+    end
+
+    def timeout_message
+      @latest_rs.timeout_message
+    end
+
     def deploy_timed_out?
       super || @latest_rs && @latest_rs.deploy_timed_out?
     end

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -2,16 +2,16 @@
 module KubernetesDeploy
   class Pod < KubernetesResource
     TIMEOUT = 10.minutes
-    SUSPICIOUS_CONTAINER_STATES = %w(ImagePullBackOff RunContainerError ErrImagePull CrashLoopBackOff).freeze
 
     def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started: nil)
       @parent = parent
       @deploy_started = deploy_started
-      @containers = definition.fetch("spec", {}).fetch("containers", {}).map { |c| c["name"] }
+      @containers = definition.fetch("spec", {}).fetch("containers", []).map { |c| Container.new(c) }
       unless @containers.present?
         logger.summary.add_paragraph("Rendered template content:\n#{definition.to_yaml}")
         raise FatalDeploymentError, "Template is missing required field spec.containers"
       end
+      @containers += definition["spec"].fetch("initContainers", []).map { |c| Container.new(c, init_container: true) }
       super(namespace: namespace, context: context, definition: definition, logger: logger)
     end
 
@@ -23,15 +23,16 @@ module KubernetesDeploy
 
       if pod_data.present?
         @found = true
-        interpret_pod_status_data(pod_data["status"], pod_data["metadata"]) # sets @phase, @status and @ready
-        if @deploy_started
-          log_suspicious_states(pod_data["status"].fetch("containerStatuses", []))
-        end
+        @phase = @status = pod_data["status"]["phase"]
+        @status += " (Reason: #{pod_data['status']['reason']})" if pod_data['status']['reason'].present?
+        @ready = ready?(pod_data["status"])
+        update_container_statuses(pod_data["status"])
       else # reset
-        @found = false
-        @phase = @status = nil
-        @ready = false
+        @found = @ready = false
+        @status = @phase = 'Unknown'
+        @containers.each(&:reset_status)
       end
+
       display_logs if unmanaged? && deploy_succeeded?
     end
 
@@ -44,13 +45,42 @@ module KubernetesDeploy
     end
 
     def deploy_failed?
-      @phase == "Failed"
+      return true if @phase == "Failed"
+      @containers.any?(&:doomed?)
     end
 
     def exists?
       @found
     end
 
+    def timeout_message
+      return if unmanaged?
+      return unless @phase == "Running" && !@ready
+      pieces = ["Your pods are running, but the following containers seem to be failing their readiness probes:"]
+      @containers.each do |c|
+        next if c.init_container? || c.ready?
+        yellow_name = ColorizedString.new(c.name).yellow
+        pieces << "> #{yellow_name} must respond with a good status code at '#{c.probe_location}'"
+      end
+      pieces.join("\n") + "\n"
+    end
+
+    def failure_message
+      doomed_containers = @containers.select(&:doomed?)
+      return unless doomed_containers.present?
+      container_messages = doomed_containers.map do |c|
+        red_name = ColorizedString.new(c.name).red
+        "> #{red_name}: #{c.doom_reason}"
+      end
+
+      intro = if unmanaged?
+        "The following containers encountered errors:"
+      else
+        "The following containers are in a state that is unlikely to be recoverable:"
+      end
+      intro + "\n" + container_messages.join("\n") + "\n"
+    end
+
     # Returns a hash in the following format:
     # {
     #   "app" => ["array of log lines", "received from app container"],
@@ -58,39 +88,35 @@ module KubernetesDeploy
     # }
     def fetch_logs
       return {} unless exists? && @containers.present?
-      @containers.each_with_object({}) do |container_name, container_logs|
+      @containers.each_with_object({}) do |container, container_logs|
         cmd = [
           "logs",
           @name,
-          "--container=#{container_name}",
+          "--container=#{container.name}",
           "--since-time=#{@deploy_started.to_datetime.rfc3339}",
         ]
         cmd << "--tail=#{LOG_LINE_COUNT}" unless unmanaged?
         out, _err, _st = kubectl.run(*cmd)
-        container_logs[container_name] = out.split("\n")
+        container_logs[container.name] = out.split("\n")
       end
     end
 
     private
 
-    def interpret_pod_status_data(status_data, metadata)
-      @status = @phase = (metadata["deletionTimestamp"] ? "Terminating" : status_data["phase"])
-
-      if @phase == "Failed" && status_data['reason'].present?
-        @status += " (Reason: #{status_data['reason']})"
-      elsif @phase != "Terminating"
-        ready_condition = status_data.fetch("conditions", []).find { |condition| condition["type"] == "Ready" }
-        @ready = ready_condition.present? && (ready_condition["status"] == "True")
-        @status += " (Ready: #{@ready})"
-      end
+    def ready?(status_data)
+      ready_condition = status_data.fetch("conditions", []).find { |condition| condition["type"] == "Ready" }
+      ready_condition.present? && (ready_condition["status"] == "True")
     end
 
-    def log_suspicious_states(container_statuses)
-      container_statuses.each do |status|
-        waiting_state = status["state"]["waiting"] if status["state"]
-        reason = waiting_state["reason"] if waiting_state
-        next unless SUSPICIOUS_CONTAINER_STATES.include?(reason)
-        @logger.warn("#{id} has container in state #{reason} (#{waiting_state['message']})")
+    def update_container_statuses(status_data)
+      @containers.each do |c|
+        key = c.init_container? ? "initContainerStatuses" : "containerStatuses"
+        if status_data.key?(key)
+          data = status_data[key].find { |st| st["name"] == c.name }
+          c.update_status(data)
+        else
+          c.reset_status
+        end
       end
     end
 
@@ -120,5 +146,69 @@ module KubernetesDeploy
 
       @already_displayed = true
     end
+
+    class Container
+      STATUS_SCAFFOLD = {
+        "state" => {
+          "running" => {},
+          "waiting" => {},
+          "terminated" => {},
+        },
+        "lastState" => {
+          "terminated" => {}
+        }
+      }.freeze
+
+      attr_reader :name, :probe_location
+
+      def initialize(definition, init_container: false)
+        @init_container = init_container
+        @name = definition["name"]
+        @image = definition["image"]
+        @probe_location = definition.fetch("readinessProbe", {}).fetch("httpGet", {})["path"]
+        @status = STATUS_SCAFFOLD.dup
+      end
+
+      def doomed?
+        doom_reason.present?
+      end
+
+      def doom_reason
+        exit_code = @status['lastState']['terminated']['exitCode']
+        last_terminated_reason = @status["lastState"]["terminated"]["reason"]
+        limbo_reason = @status["state"]["waiting"]["reason"]
+        limbo_message = @status["state"]["waiting"]["message"]
+
+        if last_terminated_reason == "ContainerCannotRun"
+          # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
+          "Failed to start (exit #{exit_code}): #{@status['lastState']['terminated']['message']}"
+        elsif limbo_reason == "CrashLoopBackOff"
+          "Crashing repeatedly (exit #{exit_code}). See logs for more information."
+        elsif %w(ImagePullBackOff ErrImagePull).include?(limbo_reason) && limbo_message.match("not found")
+          "Failed to pull image #{@image}. "\
+          "Did you wait for it to be built and pushed to the registry before deploying?"
+        elsif limbo_message == "Generate Container Config Failed"
+          # reason/message are backwards
+          # Flip this after https://github.com/kubernetes/kubernetes/commit/df41787b1a3f51b73fb6db8a2203f0a7c7c92931
+          "Failed to generate container configuration: #{limbo_reason}"
+        end
+      end
+
+      def ready?
+        @status['ready'] == "true"
+      end
+
+      def init_container?
+        @init_container
+      end
+
+      def update_status(data)
+        @status = STATUS_SCAFFOLD.deep_merge(data || {})
+      end
+
+      def reset_status
+        @status = STATUS_SCAFFOLD.dup
+      end
+    end
   end
 end

data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb

@@ -42,6 +42,14 @@ module KubernetesDeploy
       @pods.present? && @pods.all?(&:deploy_failed?)
     end
 
+    def failure_message
+      @pods.map(&:failure_message).compact.uniq.join("\n")
+    end
+
+    def timeout_message
+      @pods.map(&:timeout_message).compact.uniq.join("\n")
+    end
+
     def deploy_timed_out?
       super || @pods.present? && @pods.all?(&:deploy_timed_out?)
     end
@@ -53,7 +61,8 @@ module KubernetesDeploy
     def fetch_events
       own_events = super
       return own_events unless @pods.present?
-      own_events.merge(@pods.first.fetch_events)
+      most_useful_pod = @pods.find(&:deploy_failed?) || @pods.find(&:deploy_timed_out?) || @pods.first
+      own_events.merge(most_useful_pod.fetch_events)
     end
 
     def fetch_logs
@@ -76,7 +85,9 @@ module KubernetesDeploy
     end
 
     def container_names
-      @definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
+      regular_containers = @definition["spec"]["template"]["spec"]["containers"].map { |c| c["name"] }
+      init_containers = @definition["spec"]["template"]["spec"].fetch("initContainers", {}).map { |c| c["name"] }
+      regular_containers + init_containers
     end
 
     def find_pods(rs_data)

data/lib/kubernetes-deploy/kubernetes_resource/service.rb

@@ -6,21 +6,17 @@ module KubernetesDeploy
     def sync
       _, _err, st = kubectl.run("get", type, @name)
       @found = st.success?
-      if @found
-        endpoints, _err, st = kubectl.run("get", "endpoints", @name, "--output=jsonpath={.subsets[*].addresses[*].ip}")
-        @num_endpoints = (st.success? ? endpoints.split.length : 0)
+      @related_deployment_replicas = fetch_related_replica_count
+      @status = if @num_pods_selected = fetch_related_pod_count
+        "Selects #{@num_pods_selected} #{'pod'.pluralize(@num_pods_selected)}"
       else
-        @num_endpoints = 0
+        "Failed to count related pods"
       end
-      @status = "#{@num_endpoints} endpoints"
     end
 
     def deploy_succeeded?
-      if exposes_zero_replica_deployment?
-        @num_endpoints == 0
-      else
-        @num_endpoints > 0
-      end
+      # We can't use endpoints if we want the service to be able to fail fast when the pods are down
+      exposes_zero_replica_deployment? || selects_some_pods?
     end
 
     def deploy_failed?
@@ -28,10 +24,7 @@ module KubernetesDeploy
     end
 
     def timeout_message
-      <<-MSG.strip_heredoc.strip
-        This service does not have any endpoints. If the related pods are failing, fixing them will solve this as well.
-        If the related pods are up, this service's selector is probably incorrect.
-      MSG
+      "This service does not seem to select any pods. This means its spec.selector is probably incorrect."
     end
 
     def exists?
@@ -41,19 +34,32 @@ module KubernetesDeploy
     private
 
     def exposes_zero_replica_deployment?
-      related_deployment_replicas && related_deployment_replicas == 0
+      return false unless @related_deployment_replicas
+      @related_deployment_replicas == 0
     end
 
-    def related_deployment_replicas
-      @related_deployment_replicas ||= begin
-        selector = @definition["spec"]["selector"].map { |k, v| "#{k}=#{v}" }.join(",")
-        raw_json, _err, st = kubectl.run("get", "deployments", "--selector=#{selector}", "--output=json")
-        return unless st.success?
+    def selects_some_pods?
+      return false unless @num_pods_selected
+      @num_pods_selected > 0
+    end
 
-        deployments = JSON.parse(raw_json)["items"]
-        return unless deployments.length == 1
-        deployments.first["spec"]["replicas"].to_i
-      end
+    def selector
+      @selector ||= @definition["spec"]["selector"].map { |k, v| "#{k}=#{v}" }.join(",")
+    end
+
+    def fetch_related_pod_count
+      raw_json, _err, st = kubectl.run("get", "pods", "--selector=#{selector}", "--output=json")
+      return unless st.success?
+      JSON.parse(raw_json)["items"].length
+    end
+
+    def fetch_related_replica_count
+      raw_json, _err, st = kubectl.run("get", "deployments", "--selector=#{selector}", "--output=json")
+      return unless st.success?
+
+      deployments = JSON.parse(raw_json)["items"]
+      return unless deployments.length == 1
+      deployments.first["spec"]["replicas"].to_i
     end
   end
 end

data/lib/kubernetes-deploy/runner.rb

@@ -157,6 +157,7 @@ module KubernetesDeploy
 
       if success_count > 0
         @logger.summary.add_action("successfully deployed #{success_count} #{'resource'.pluralize(success_count)}")
+        successful_resources.map(&:sync) # make sure we're printing the latest on resources that succeeded early
         final_statuses = successful_resources.map(&:pretty_status).join("\n")
         @logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
       end

data/lib/kubernetes-deploy/version.rb

@@ -1,3 +1,3 @@
 module KubernetesDeploy
-  VERSION = "0.8.3"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.9.0
 platform: ruby
 authors:
 - Katrina Verey