kubernetes-deploy 0.6.6 → 0.7.0

data/lib/kubernetes-deploy/kubernetes_resource/service.rb

@@ -3,24 +3,16 @@ module KubernetesDeploy
   class Service < KubernetesResource
     TIMEOUT = 5.minutes
 
-    def initialize(name, namespace, context, file)
-      @name = name
-      @namespace = namespace
-      @context = context
-      @file = file
-    end
-
     def sync
-      _, _err, st = run_kubectl("get", type, @name)
+      _, _err, st = kubectl.run("get", type, @name)
       @found = st.success?
       if @found
-        endpoints, _err, st = run_kubectl("get", "endpoints", @name, "--output=jsonpath={.subsets[*].addresses[*].ip}")
+        endpoints, _err, st = kubectl.run("get", "endpoints", @name, "--output=jsonpath={.subsets[*].addresses[*].ip}")
         @num_endpoints = (st.success? ? endpoints.split.length : 0)
       else
         @num_endpoints = 0
       end
       @status = "#{@num_endpoints} endpoints"
-      log_status
     end
 
     def deploy_succeeded?
@@ -31,6 +23,13 @@ module KubernetesDeploy
       false
     end
 
+    def timeout_message
+      <<-MSG.strip_heredoc.strip
+        This service does not have any endpoints. If the related pods are failing, fixing them will solve this as well.
+        If the related pods are up, this service's selector is probably incorrect.
+      MSG
+    end
+
     def exists?
       @found
     end

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -1,37 +1,49 @@
 # frozen_string_literal: true
 module KubernetesDeploy
   class ResourceWatcher
-    def initialize(resources)
+    def initialize(resources, logger:)
       unless resources.is_a?(Enumerable)
         raise ArgumentError, <<-MSG.strip
 ResourceWatcher expects Enumerable collection, got `#{resources.class}` instead
 MSG
       end
       @resources = resources
+      @logger = logger
     end
 
-    def run(delay_sync: 3.seconds, logger: KubernetesDeploy.logger)
+    def run(delay_sync: 3.seconds)
       delay_sync_until = Time.now.utc
       started_at = delay_sync_until
-      human_resources = @resources.map(&:id).join(", ")
-      max_wait_time = @resources.map(&:timeout).max
-      logger.info("Waiting for #{human_resources} with #{max_wait_time}s timeout")
 
       while @resources.present?
         if Time.now.utc < delay_sync_until
           sleep(delay_sync_until - Time.now.utc)
         end
+        watch_time = (Time.now.utc - started_at).round(1)
         delay_sync_until = Time.now.utc + delay_sync # don't pummel the API if the sync is fast
         @resources.each(&:sync)
         newly_finished_resources, @resources = @resources.partition(&:deploy_finished?)
+
+        new_success_list = []
         newly_finished_resources.each do |resource|
-          next unless resource.deploy_failed? || resource.deploy_timed_out?
-          logger.error("#{resource.id} failed to deploy with status '#{resource.status}'.")
+          if resource.deploy_failed?
+            @logger.error("#{resource.id} failed to deploy after #{watch_time}s")
+          elsif resource.deploy_timed_out?
+            @logger.error("#{resource.id} deployment timed out")
+          else
+            new_success_list << resource.id
+          end
+        end
+
+        unless new_success_list.empty?
+          success_string = ColorizedString.new("Successfully deployed in #{watch_time}s:").green
+          @logger.info("#{success_string} #{new_success_list.join(', ')}")
         end
-      end
 
-      watch_time = Time.now.utc - started_at
-      logger.info("Spent #{watch_time.round(2)}s waiting for #{human_resources}")
+        if newly_finished_resources.present? && @resources.present? # something happened this cycle, more to go
+          @logger.info("Continuing to wait for: #{@resources.map(&:id).join(', ')}")
+        end
+      end
     end
   end
 end

data/lib/kubernetes-deploy/restart_task.rb

@@ -1,11 +1,9 @@
 # frozen_string_literal: true
 require 'kubernetes-deploy/kubeclient_builder'
-require 'kubernetes-deploy/ui_helpers'
 require 'kubernetes-deploy/resource_watcher'
 
 module KubernetesDeploy
   class RestartTask
-    include UIHelpers
     include KubernetesDeploy::KubeclientBuilder
 
     class DeploymentNotFoundError < FatalDeploymentError
@@ -25,7 +23,7 @@ module KubernetesDeploy
     HTTP_OK_RANGE = 200..299
     ANNOTATION = "shipit.shopify.io/restart"
 
-    def initialize(context:, namespace:, logger: KubernetesDeploy.logger)
+    def initialize(context:, namespace:, logger:)
       @context = context
       @namespace = namespace
       @logger = logger
@@ -35,6 +33,7 @@ module KubernetesDeploy
     end
 
     def perform(deployments_names = nil)
+      @logger.reset
       verify_namespace
 
       if deployments_names
@@ -53,21 +52,27 @@ module KubernetesDeploy
         end
       end
 
-      phase_heading("Triggering restart by touching ENV[RESTARTED_AT]")
+      @logger.phase_heading("Triggering restart by touching ENV[RESTARTED_AT]")
       patch_kubeclient_deployments(deployments)
 
-      phase_heading("Waiting for rollout")
+      @logger.phase_heading("Waiting for rollout")
       wait_for_rollout(deployments)
 
       names = deployments.map { |d| "`#{d.metadata.name}`" }
       @logger.info "Restart of #{names.sort.join(', ')} deployments succeeded"
+      true
+    rescue FatalDeploymentError => error
+      @logger.fatal "#{error.class}: #{error.message}"
+      false
     end
 
     private
 
     def wait_for_rollout(kubeclient_resources)
-      resources = kubeclient_resources.map { |d| Deployment.new(d.metadata.name, @namespace, @context, nil) }
-      watcher = ResourceWatcher.new(resources)
+      resources = kubeclient_resources.map do |d|
+        Deployment.new(name: d.metadata.name, namespace: @namespace, context: @context, file: nil, logger: @logger)
+      end
+      watcher = ResourceWatcher.new(resources, logger: @logger)
       watcher.run
     end
 

data/lib/kubernetes-deploy/runner.rb

@@ -22,14 +22,12 @@ require 'kubernetes-deploy/kubernetes_resource'
   require "kubernetes-deploy/kubernetes_resource/#{subresource}"
 end
 require 'kubernetes-deploy/resource_watcher'
-require "kubernetes-deploy/ui_helpers"
 require 'kubernetes-deploy/kubectl'
 require 'kubernetes-deploy/kubeclient_builder'
 require 'kubernetes-deploy/ejson_secret_provisioner'
 
 module KubernetesDeploy
   class Runner
-    include UIHelpers
     include KubeclientBuilder
 
     PREDEPLOY_SEQUENCE = %w(
@@ -68,71 +66,63 @@ module KubernetesDeploy
     PRUNE_WHITELIST_V_1_5 = %w(extensions/v1beta1/HorizontalPodAutoscaler).freeze
     PRUNE_WHITELIST_V_1_6 = %w(autoscaling/v1/HorizontalPodAutoscaler).freeze
 
-    def self.with_friendly_errors
-      yield
-    rescue FatalDeploymentError => error
-      KubernetesDeploy.logger.fatal <<-MSG
-#{error.class}: #{error.message}
-  #{error.backtrace && error.backtrace.join("\n  ")}
-MSG
-      exit 1
-    end
-
-    def initialize(namespace:, current_sha:, context:, template_dir:,
-      wait_for_completion:, allow_protected_ns: false, prune: true, bindings: {})
+    def initialize(namespace:, context:, current_sha:, template_dir:, logger:, bindings: {})
       @namespace = namespace
       @context = context
       @current_sha = current_sha
       @template_dir = File.expand_path(template_dir)
+      @logger = logger
+      @bindings = bindings
       # Max length of podname is only 63chars so try to save some room by truncating sha to 8 chars
       @id = current_sha[0...8] + "-#{SecureRandom.hex(4)}" if current_sha
-      @wait_for_completion = wait_for_completion
-      @allow_protected_ns = allow_protected_ns
-      @prune = prune
-      @bindings = bindings
     end
 
-    def wait_for_completion?
-      @wait_for_completion
-    end
+    def run(verify_result: true, allow_protected_ns: false, prune: true)
+      @logger.reset
 
-    def allow_protected_ns?
-      @allow_protected_ns
-    end
-
-    def run
-      phase_heading("Validating configuration")
-      validate_configuration
-
-      phase_heading("Identifying deployment target")
+      @logger.phase_heading("Initializing deploy")
+      validate_configuration(allow_protected_ns: allow_protected_ns, prune: prune)
       confirm_context_exists
       confirm_namespace_exists
-
-      phase_heading("Parsing deploy content")
       resources = discover_resources
 
-      phase_heading("Checking initial resource statuses")
+      @logger.phase_heading("Checking initial resource statuses")
       resources.each(&:sync)
-
-      ejson = EjsonSecretProvisioner.new(namespace: @namespace, context: @context, template_dir: @template_dir)
+      resources.each { |r| @logger.info(r.pretty_status) }
+
+      ejson = EjsonSecretProvisioner.new(
+        namespace: @namespace,
+        context: @context,
+        template_dir: @template_dir,
+        logger: @logger
+      )
       if ejson.secret_changes_required?
-        phase_heading("Deploying kubernetes secrets from #{EjsonSecretProvisioner::EJSON_SECRETS_FILE}")
+        @logger.phase_heading("Deploying kubernetes secrets from #{EjsonSecretProvisioner::EJSON_SECRETS_FILE}")
         ejson.run
       end
 
-      phase_heading("Predeploying priority resources")
-      predeploy_priority_resources(resources)
+      if deploy_has_priority_resources?(resources)
+        @logger.phase_heading("Predeploying priority resources")
+        predeploy_priority_resources(resources)
+      end
 
-      phase_heading("Deploying all resources")
-      if PROTECTED_NAMESPACES.include?(@namespace) && @prune
+      @logger.phase_heading("Deploying all resources")
+      if PROTECTED_NAMESPACES.include?(@namespace) && prune
         raise FatalDeploymentError, "Refusing to deploy to protected namespace '#{@namespace}' with pruning enabled"
       end
 
-      deploy_resources(resources, prune: @prune)
+      deploy_resources(resources, prune: prune)
 
-      return unless wait_for_completion?
+      return true unless verify_result
       wait_for_completion(resources)
-      report_final_status(resources)
+      record_statuses(resources)
+      success = resources.all?(&:deploy_succeeded?)
+    rescue FatalDeploymentError => error
+      @logger.summary.add_action(error.message)
+      success = false
+    ensure
+      @logger.print_summary(success)
+      success
     end
 
     def template_variables
@@ -144,6 +134,23 @@ MSG
 
     private
 
+    def record_statuses(resources)
+      successful_resources, failed_resources = resources.partition(&:deploy_succeeded?)
+      fail_count = failed_resources.length
+      success_count = successful_resources.length
+
+      if success_count > 0
+        @logger.summary.add_action("successfully deployed #{success_count} #{'resource'.pluralize(success_count)}")
+        final_statuses = successful_resources.map(&:pretty_status).join("\n")
+        @logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
+      end
+
+      if fail_count > 0
+        @logger.summary.add_action("failed to deploy #{fail_count} #{'resource'.pluralize(fail_count)}")
+        failed_resources.each { |r| @logger.summary.add_paragraph(r.debug_message) }
+      end
+    end
+
     def versioned_prune_whitelist
       if server_major_version == "1.5"
         BASE_PRUNE_WHITELIST + PRUNE_WHITELIST_V_1_5
@@ -154,7 +161,7 @@ MSG
 
     def server_major_version
       @server_major_version ||= begin
-        out, _, _ = run_kubectl('version', '--short')
+        out, _, _ = kubectl.run('version', '--short')
         matchdata = /Server Version: v(?<version>\d\.\d)/.match(out)
         raise "Could not determine server version" unless matchdata[:version]
         matchdata[:version]
@@ -163,7 +170,7 @@ MSG
 
     # Inspect the file referenced in the kubectl stderr
     # to make it easier for developer to understand what's going on
-    def inspect_kubectl_out_for_files(stderr)
+    def find_bad_file_from_kubectl_output(stderr)
       # Output example:
       # Error from server (BadRequest): error when creating "/path/to/configmap-gqq5oh.yml20170411-33615-t0t3m":
       match = stderr.match(%r{BadRequest.*"(?<path>\/\S+\.yml\S+)"})
@@ -172,12 +179,12 @@ MSG
       path = match[:path]
       if path.present? && File.file?(path)
         suspicious_file = File.read(path)
-        KubernetesDeploy.logger.warn("Inspecting the file mentioned in the error message (#{path})")
-        KubernetesDeploy.logger.warn(suspicious_file)
-      else
-        KubernetesDeploy.logger.warn("Detected a file (#{path.inspect}) referenced in the kubectl stderr " \
-          "but was unable to inspect it")
       end
+      [File.basename(path, ".*"), suspicious_file]
+    end
+
+    def deploy_has_priority_resources?(resources)
+      resources.any? { |r| PREDEPLOY_SEQUENCE.include?(r.type) }
     end
 
     def predeploy_priority_resources(resource_list)
@@ -186,31 +193,52 @@ MSG
         next if matching_resources.empty?
         deploy_resources(matching_resources)
         wait_for_completion(matching_resources)
-        fail_list = matching_resources.select { |r| r.deploy_failed? || r.deploy_timed_out? }.map(&:id)
-        unless fail_list.empty?
-          raise FatalDeploymentError, "The following priority resources failed to deploy: #{fail_list.join(', ')}"
+
+        failed_resources = matching_resources.reject(&:deploy_succeeded?)
+        fail_count = failed_resources.length
+        if fail_count > 0
+          failed_resources.each { |r| @logger.summary.add_paragraph(r.debug_message) }
+          raise FatalDeploymentError, "Failed to deploy #{fail_count} priority #{'resource'.pluralize(fail_count)}"
         end
+        @logger.blank_line
       end
     end
 
     def discover_resources
       resources = []
+      @logger.info("Discovering templates:")
       Dir.foreach(@template_dir) do |filename|
         next unless filename.end_with?(".yml.erb", ".yml")
 
         split_templates(filename) do |tempfile|
           resource_id = discover_resource_via_dry_run(tempfile)
           type, name = resource_id.split("/", 2) # e.g. "pod/web-198612918-dzvfb"
-          resources << KubernetesResource.for_type(type, name, @namespace, @context, tempfile)
-          KubernetesDeploy.logger.info "Discovered template for #{resource_id}"
+          resources << KubernetesResource.for_type(type: type, name: name, namespace: @namespace, context: @context,
+            file: tempfile, logger: @logger)
+          @logger.info "  - #{resource_id}"
         end
       end
       resources
     end
 
     def discover_resource_via_dry_run(tempfile)
-      resource_id, _err, st = run_kubectl("create", "-f", tempfile.path, "--dry-run", "--output=name")
-      raise FatalDeploymentError, "Dry run failed for template #{File.basename(tempfile.path)}." unless st.success?
+      command = ["create", "-f", tempfile.path, "--dry-run", "--output=name"]
+      resource_id, err, st = kubectl.run(*command, log_failure: false)
+
+      unless st.success?
+        debug_msg = <<-DEBUG_MSG.strip_heredoc
+          This usually means template '#{File.basename(tempfile.path, '.*')}' is not a valid Kubernetes template.
+
+          Error from kubectl:
+            #{err}
+
+          Rendered template content:
+        DEBUG_MSG
+        debug_msg += File.read(tempfile.path)
+        @logger.summary.add_paragraph(debug_msg)
+
+        raise FatalDeploymentError, "Kubectl dry run failed (command: #{Shellwords.join(command)})"
+      end
       resource_id
     end
 
@@ -226,21 +254,42 @@ MSG
         yield f
       end
     rescue Psych::SyntaxError => e
-      KubernetesDeploy.logger.error(rendered_content)
-      raise FatalDeploymentError, "Template #{filename} cannot be parsed: #{e.message}"
+      debug_msg = <<-INFO.strip_heredoc
+        Error message: #{e}
+
+        Template content:
+        ---
+      INFO
+      debug_msg += rendered_content
+      @logger.summary.add_paragraph(debug_msg)
+      raise FatalDeploymentError, "Template '#{filename}' cannot be parsed"
     end
 
-    def report_final_status(resources)
-      if resources.all?(&:deploy_succeeded?)
-        log_green("Deploy succeeded!")
+    def record_apply_failure(err)
+      file_name, file_content = find_bad_file_from_kubectl_output(err)
+      if file_name
+        debug_msg = <<-HELPFUL_MESSAGE.strip_heredoc
+          This usually means your template named '#{file_name}' is invalid.
+
+          Error from kubectl:
+            #{err}
+
+          Rendered template content:
+        HELPFUL_MESSAGE
+        debug_msg += file_content || "Failed to read file"
       else
-        fail_list = resources.select { |r| r.deploy_failed? || r.deploy_timed_out? }.map(&:id)
-        raise FatalDeploymentError, "The following resources failed to deploy: #{fail_list.join(', ')}"
+        debug_msg = <<-FALLBACK_MSG
+          This usually means one of your templates is invalid, but we were unable to automatically identify which one.
+          Please inspect the error message from kubectl:
+            #{err}
+        FALLBACK_MSG
       end
+
+      @logger.summary.add_paragraph(debug_msg)
     end
 
     def wait_for_completion(watched_resources)
-      watcher = ResourceWatcher.new(watched_resources)
+      watcher = ResourceWatcher.new(watched_resources, logger: @logger)
       watcher.run
     end
 
@@ -253,9 +302,12 @@ MSG
         erb_binding.local_variable_set(var_name, value)
       end
       erb_template.result(erb_binding)
+    rescue NameError => e
+      @logger.summary.add_paragraph("Error from renderer:\n  #{e.message.tr("\n", ' ')}")
+      raise FatalDeploymentError, "Template '#{filename}' cannot be rendered"
     end
 
-    def validate_configuration
+    def validate_configuration(allow_protected_ns:, prune:)
       errors = []
       if ENV["KUBECONFIG"].blank? || !File.file?(ENV["KUBECONFIG"])
         errors << "Kube config not found at #{ENV['KUBECONFIG']}"
@@ -274,15 +326,13 @@ MSG
       if @namespace.blank?
         errors << "Namespace must be specified"
       elsif PROTECTED_NAMESPACES.include?(@namespace)
-        if allow_protected_ns? && @prune
+        if allow_protected_ns && prune
           errors << "Refusing to deploy to protected namespace '#{@namespace}' with pruning enabled"
-        elsif allow_protected_ns?
-          warning = <<-WARNING.strip_heredoc
-          You're deploying to protected namespace #{@namespace}, which cannot be pruned.
-          Existing resources can only be removed manually with kubectl. Removing templates from the set deployed will have no effect.
-          ***Please do not deploy to #{@namespace} unless you really know what you are doing.***
-          WARNING
-          KubernetesDeploy.logger.warn(warning)
+        elsif allow_protected_ns
+          @logger.warn("You're deploying to protected namespace #{@namespace}, which cannot be pruned.")
+          @logger.warn("Existing resources can only be removed manually with kubectl. " \
+            "Removing templates from the set deployed will have no effect.")
+          @logger.warn("***Please do not deploy to #{@namespace} unless you really know what you are doing.***")
         else
           errors << "Refusing to deploy to protected namespace '#{@namespace}'"
         end
@@ -292,38 +342,41 @@ MSG
         errors << "Context must be specified"
       end
 
-      raise FatalDeploymentError, "Configuration invalid: #{errors.join(', ')}" unless errors.empty?
-      KubernetesDeploy.logger.info("All required parameters and files are present")
+      unless errors.empty?
+        @logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
+        raise FatalDeploymentError, "Configuration invalid"
+      end
+
+      @logger.info("All required parameters and files are present")
     end
 
     def deploy_resources(resources, prune: false)
-      KubernetesDeploy.logger.info("Deploying resources:")
+      @logger.info("Deploying resources:")
 
       # Apply can be done in one large batch, the rest have to be done individually
       applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
 
       individuals.each do |r|
-        KubernetesDeploy.logger.info("- #{r.id}")
+        @logger.info("- #{r.id}")
         r.deploy_started = Time.now.utc
         case r.deploy_method
         when :replace
-          _, _, st = run_kubectl("replace", "-f", r.file.path)
+          _, _, st = kubectl.run("replace", "-f", r.file.path, log_failure: false)
         when :replace_force
-          _, _, st = run_kubectl("replace", "--force", "-f", r.file.path)
+          _, _, st = kubectl.run("replace", "--force", "-f", r.file.path, log_failure: false)
         else
           # Fail Fast! This is a programmer mistake.
           raise ArgumentError, "Unexpected deploy method! (#{r.deploy_method.inspect})"
         end
 
+        next if st.success?
+        # it doesn't exist so we can't replace it
+        _, err, st = kubectl.run("create", "-f", r.file.path, log_failure: false)
         unless st.success?
-          # it doesn't exist so we can't replace it
-          _, err, st = run_kubectl("create", "-f", r.file.path)
-          unless st.success?
-            raise FatalDeploymentError, <<-MSG.strip_heredoc
-              Failed to replace or create resource: #{r.id}
-              #{err}
-            MSG
-          end
+          raise FatalDeploymentError, <<-MSG.strip_heredoc
+            Failed to replace or create resource: #{r.id}
+            #{err}
+          MSG
         end
       end
 
@@ -335,7 +388,7 @@ MSG
 
       command = ["apply"]
       resources.each do |r|
-        KubernetesDeploy.logger.info("- #{r.id}")
+        @logger.info("- #{r.id} (timeout: #{r.timeout}s)")
         command.push("-f", r.file.path)
         r.deploy_started = Time.now.utc
       end
@@ -345,43 +398,43 @@ MSG
         versioned_prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
       end
 
-      _, err, st = run_kubectl(*command)
-      unless st.success?
-        inspect_kubectl_out_for_files(err)
-        raise FatalDeploymentError, <<-MSG
-"The following command failed: #{Shellwords.join(command)}"
-#{err}
-MSG
+      out, err, st = kubectl.run(*command, log_failure: false)
+      if st.success?
+        log_pruning(out) if prune
+      else
+        record_apply_failure(err)
+        raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
       end
     end
 
+    def log_pruning(kubectl_output)
+      pruned = kubectl_output.scan(/^(.*) pruned$/)
+      return unless pruned.present?
+
+      @logger.info("The following resources were pruned: #{pruned.join(', ')}")
+      @logger.summary.add_action("pruned #{pruned.length} resources")
+    end
+
     def confirm_context_exists
-      out, err, st = run_kubectl("config", "get-contexts", "-o", "name", namespaced: false, with_context: false)
+      out, err, st = kubectl.run("config", "get-contexts", "-o", "name",
+        use_namespace: false, use_context: false, log_failure: false)
       available_contexts = out.split("\n")
       if !st.success?
         raise FatalDeploymentError, err
       elsif !available_contexts.include?(@context)
         raise FatalDeploymentError, "Context #{@context} is not available. Valid contexts: #{available_contexts}"
       end
-      KubernetesDeploy.logger.info("Context #{@context} found")
+      @logger.info("Context #{@context} found")
     end
 
     def confirm_namespace_exists
-      _, _, st = run_kubectl("get", "namespace", @namespace, namespaced: false)
+      _, _, st = kubectl.run("get", "namespace", @namespace, use_namespace: false, log_failure: false)
       raise FatalDeploymentError, "Namespace #{@namespace} not found" unless st.success?
-      KubernetesDeploy.logger.info("Namespace #{@namespace} found")
+      @logger.info("Namespace #{@namespace} found")
     end
 
-    def run_kubectl(*args, namespaced: true, with_context: true)
-      if namespaced
-        raise KubectlError, "Namespace missing for namespaced command" if @namespace.blank?
-      end
-
-      if with_context
-        raise KubectlError, "Explicit context is required to run this command" if @context.blank?
-      end
-
-      Kubectl.run_kubectl(*args, namespace: @namespace, context: @context)
+    def kubectl
+      @kubectl ||= Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: true)
     end
   end
 end