kubernetes-deploy 0.30.0 → 0.31.0

data/lib/{kubernetes-deploy → krane}/renderer.rb

@@ -5,7 +5,7 @@ require 'securerandom'
 require 'yaml'
 require 'json'
 
-module KubernetesDeploy
+module Krane
   class Renderer
     class InvalidPartialError < InvalidTemplateError
       attr_accessor :parents, :content, :filename

data/lib/{kubernetes-deploy → krane}/resource_cache.rb

@@ -2,7 +2,7 @@
 
 require 'concurrent/hash'
 
-module KubernetesDeploy
+module Krane
   class ResourceCache
     delegate :namespace, :context, :logger, to: :@task_config
 
@@ -17,7 +17,7 @@ module KubernetesDeploy
     def get_instance(kind, resource_name, raise_if_not_found: false)
       instance = use_or_populate_cache(kind).fetch(resource_name, {})
       if instance.blank? && raise_if_not_found
-        raise KubernetesDeploy::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for kind #{kind})"
+        raise Krane::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for kind #{kind})"
       end
       instance
     rescue KubectlError
@@ -51,9 +51,10 @@ module KubernetesDeploy
 
     def fetch_by_kind(kind)
       resource_class = KubernetesResource.class_for_kind(kind)
+      global_kind = @task_config.global_kinds.map(&:downcase).include?(kind.downcase)
       output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT
       raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json",
-         output_is_sensitive: output_is_sensitive)
+         output_is_sensitive: output_is_sensitive, use_namespace: !global_kind)
       raise KubectlError unless st.success?
 
       instances = {}

data/lib/krane/resource_deployer.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require 'krane/resource_watcher'
+require 'krane/concerns/template_reporting'
+
+module Krane
+  class ResourceDeployer
+    extend Krane::StatsD::MeasureMethods
+    include Krane::TemplateReporting
+
+    delegate :logger, to: :@task_config
+    attr_reader :statsd_tags
+
+    def initialize(task_config:, prune_whitelist:, max_watch_seconds:, current_sha: nil, selector:, statsd_tags:)
+      @task_config = task_config
+      @prune_whitelist = prune_whitelist
+      @max_watch_seconds = max_watch_seconds
+      @current_sha = current_sha
+      @selector = selector
+      @statsd_tags = statsd_tags
+    end
+
+    def deploy!(resources, verify_result, prune)
+      if verify_result
+        deploy_all_resources(resources, prune: prune, verify: true)
+        failed_resources = resources.reject(&:deploy_succeeded?)
+        success = failed_resources.empty?
+        if !success && failed_resources.all?(&:deploy_timed_out?)
+          raise DeploymentTimeoutError
+        end
+        raise FatalDeploymentError unless success
+      else
+        deploy_all_resources(resources, prune: prune, verify: false)
+        logger.summary.add_action("deployed #{resources.length} #{'resource'.pluralize(resources.length)}")
+        warning = <<~MSG
+          Deploy result verification is disabled for this deploy.
+          This means the desired changes were communicated to Kubernetes, but the deploy did not make sure they actually succeeded.
+        MSG
+        logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
+      end
+    end
+
+    def predeploy_priority_resources(resource_list, predeploy_sequence)
+      bare_pods = resource_list.select { |resource| resource.is_a?(Pod) }
+      if bare_pods.count == 1
+        bare_pods.first.stream_logs = true
+      end
+
+      predeploy_sequence.each do |resource_type|
+        matching_resources = resource_list.select { |r| r.type == resource_type }
+        next if matching_resources.empty?
+        deploy_resources(matching_resources, verify: true, record_summary: false)
+
+        failed_resources = matching_resources.reject(&:deploy_succeeded?)
+        fail_count = failed_resources.length
+        if fail_count > 0
+          Krane::Concurrency.split_across_threads(failed_resources) do |r|
+            r.sync_debug_info(kubectl)
+          end
+          failed_resources.each { |r| logger.summary.add_paragraph(r.debug_message) }
+          raise FatalDeploymentError, "Failed to deploy #{fail_count} priority #{'resource'.pluralize(fail_count)}"
+        end
+        logger.blank_line
+      end
+    end
+    measure_method(:predeploy_priority_resources, 'priority_resources.duration')
+
+    private
+
+    def deploy_all_resources(resources, prune: false, verify:, record_summary: true)
+      deploy_resources(resources, prune: prune, verify: verify, record_summary: record_summary)
+    end
+    measure_method(:deploy_all_resources, 'normal_resources.duration')
+
+    def deploy_resources(resources, prune: false, verify:, record_summary: true)
+      return if resources.empty?
+      deploy_started_at = Time.now.utc
+
+      if resources.length > 1
+        logger.info("Deploying resources:")
+        resources.each do |r|
+          logger.info("- #{r.id} (#{r.pretty_timeout_type})")
+        end
+      else
+        resource = resources.first
+        logger.info("Deploying #{resource.id} (#{resource.pretty_timeout_type})")
+      end
+
+      # Apply can be done in one large batch, the rest have to be done individually
+      applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
+      # Prunable resources should also applied so that they can  be pruned
+      pruneable_types = @prune_whitelist.map { |t| t.split("/").last }
+      applyables += individuals.select { |r| pruneable_types.include?(r.type) }
+
+      individuals.each do |individual_resource|
+        individual_resource.deploy_started_at = Time.now.utc
+
+        case individual_resource.deploy_method
+        when :create
+          err, status = create_resource(individual_resource)
+        when :replace
+          err, status = replace_or_create_resource(individual_resource)
+        when :replace_force
+          err, status = replace_or_create_resource(individual_resource, force: true)
+        else
+          # Fail Fast! This is a programmer mistake.
+          raise ArgumentError, "Unexpected deploy method! (#{individual_resource.deploy_method.inspect})"
+        end
+
+        next if status.success?
+
+        raise FatalDeploymentError, <<~MSG
+          Failed to replace or create resource: #{individual_resource.id}
+          #{individual_resource.sensitive_template_content? ? '<suppressed sensitive output>' : err}
+        MSG
+      end
+
+      apply_all(applyables, prune)
+
+      if verify
+        watcher = Krane::ResourceWatcher.new(resources: resources, deploy_started_at: deploy_started_at,
+          timeout: @max_watch_seconds, task_config: @task_config, sha: @current_sha)
+        watcher.run(record_summary: record_summary)
+      end
+    end
+
+    def apply_all(resources, prune)
+      return unless resources.present?
+      command = %w(apply)
+
+      Dir.mktmpdir do |tmp_dir|
+        resources.each do |r|
+          FileUtils.symlink(r.file_path, tmp_dir)
+          r.deploy_started_at = Time.now.utc
+        end
+        command.push("-f", tmp_dir)
+
+        if prune && @prune_whitelist.present?
+          command.push("--prune")
+          if @selector
+            command.push("--selector", @selector.to_s)
+          else
+            command.push("--all")
+          end
+          @prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
+        end
+
+        output_is_sensitive = resources.any?(&:sensitive_template_content?)
+        global_mode = resources.all?(&:global?)
+        out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive,
+          use_namespace: !global_mode)
+
+        if st.success?
+          log_pruning(out) if prune
+        else
+          record_apply_failure(err, resources: resources)
+          raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
+        end
+      end
+    end
+    measure_method(:apply_all)
+
+    def log_pruning(kubectl_output)
+      pruned = kubectl_output.scan(/^(.*) pruned$/)
+      return unless pruned.present?
+
+      logger.info("The following resources were pruned: #{pruned.join(', ')}")
+      logger.summary.add_action("pruned #{pruned.length} #{'resource'.pluralize(pruned.length)}")
+    end
+
+    def record_apply_failure(err, resources: [])
+      warn_msg = "WARNING: Any resources not mentioned in the error(s) below were likely created/updated. " \
+        "You may wish to roll back this deploy."
+      logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
+
+      unidentified_errors = []
+      filenames_with_sensitive_content = resources
+        .select(&:sensitive_template_content?)
+        .map { |r| File.basename(r.file_path) }
+
+      server_dry_run_validated_resource = resources
+        .select(&:server_dry_run_validated?)
+        .map { |r| File.basename(r.file_path) }
+
+      err.each_line do |line|
+        bad_files = find_bad_files_from_kubectl_output(line)
+        unless bad_files.present?
+          unidentified_errors << line
+          next
+        end
+
+        bad_files.each do |f|
+          err_msg = f[:err]
+          if filenames_with_sensitive_content.include?(f[:filename])
+            # Hide the error and template contents in case it has sensitive information
+            # we display full error messages as we assume there's no sensitive info leak after server-dry-run
+            err_msg = "SUPPRESSED FOR SECURITY" unless server_dry_run_validated_resource.include?(f[:filename])
+            record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: nil)
+          else
+            record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: f[:content])
+          end
+        end
+      end
+      return unless unidentified_errors.any?
+
+      if (filenames_with_sensitive_content - server_dry_run_validated_resource).present?
+        warn_msg = "WARNING: There was an error applying some or all resources. The raw output may be sensitive and " \
+          "so cannot be displayed."
+        logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
+      else
+        heading = ColorizedString.new('Unidentified error(s):').red
+        msg = FormattedLogger.indent_four(unidentified_errors.join)
+        logger.summary.add_paragraph("#{heading}\n#{msg}")
+      end
+    end
+
+    def replace_or_create_resource(resource, force: false)
+      args = if force
+        ["replace", "--force", "--cascade", "-f", resource.file_path]
+      else
+        ["replace", "-f", resource.file_path]
+      end
+
+      _, err, status = kubectl.run(*args, log_failure: false, output_is_sensitive: resource.sensitive_template_content?,
+        raise_if_not_found: true, use_namespace: !resource.global?)
+
+      [err, status]
+    rescue Krane::Kubectl::ResourceNotFoundError
+      # it doesn't exist so we can't replace it, we try to create it
+      create_resource(resource)
+    end
+
+    def create_resource(resource)
+      out, err, status = kubectl.run("create", "-f", resource.file_path, log_failure: false,
+        output: 'json', output_is_sensitive: resource.sensitive_template_content?,
+        use_namespace: !resource.global?)
+
+      # For resources that rely on a generateName attribute, we get the `name` from the result of the call to `create`
+      # We must explicitly set this name value so that the `apply` step for pruning can run successfully
+      if status.success? && resource.uses_generate_name?
+        resource.use_generated_name(JSON.parse(out))
+      end
+
+      [err, status]
+    end
+
+    # Inspect the file referenced in the kubectl stderr
+    # to make it easier for developer to understand what's going on
+    def find_bad_files_from_kubectl_output(line)
+      # stderr often contains one or more lines like the following, from which we can extract the file path(s):
+      # Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
+
+      line.scan(%r{"(/\S+\.ya?ml\S*)"}).each_with_object([]) do |matches, bad_files|
+        matches.each do |path|
+          content = File.read(path) if File.file?(path)
+          bad_files << { filename: File.basename(path), err: line, content: content }
+        end
+      end
+    end
+
+    def kubectl
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
+    end
+  end
+end

data/lib/{kubernetes-deploy → krane}/resource_watcher.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require 'kubernetes-deploy/concurrency'
-require 'kubernetes-deploy/resource_cache'
+require 'krane/concurrency'
+require 'krane/resource_cache'
 
-module KubernetesDeploy
+module Krane
   class ResourceWatcher
-    extend KubernetesDeploy::StatsD::MeasureMethods
+    extend Krane::StatsD::MeasureMethods
     delegate :namespace, :context, :logger, to: :@task_config
 
     def initialize(resources:, task_config:, deploy_started_at: Time.now.utc,
@@ -53,7 +53,7 @@ module KubernetesDeploy
 
     def sync_resources(resources)
       cache = ResourceCache.new(@task_config)
-      KubernetesDeploy::Concurrency.split_across_threads(resources) { |r| r.sync(cache) }
+      Krane::Concurrency.split_across_threads(resources) { |r| r.sync(cache) }
       resources.each(&:after_sync)
     end
     measure_method(:sync_resources, "sync.duration")
@@ -141,7 +141,7 @@ module KubernetesDeploy
         end
 
         kubectl = Kubectl.new(task_config: @task_config, log_failure_by_default: false)
-        KubernetesDeploy::Concurrency.split_across_threads(failed_resources + global_timeouts) do |r|
+        Krane::Concurrency.split_across_threads(failed_resources + global_timeouts) do |r|
           r.sync_debug_info(kubectl)
         end
 

data/lib/krane/restart_task.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+require 'krane/common'
+require 'krane/kubernetes_resource'
+require 'krane/kubernetes_resource/deployment'
+require 'krane/kubeclient_builder'
+require 'krane/resource_watcher'
+require 'krane/kubectl'
+
+module Krane
+  # Restart the pods in one or more deployments
+  class RestartTask
+    class FatalRestartError < FatalDeploymentError; end
+
+    class RestartAPIError < FatalRestartError
+      def initialize(deployment_name, response)
+        super("Failed to restart #{deployment_name}. " \
+            "API returned non-200 response code (#{response.code})\n" \
+            "Response:\n#{response.body}")
+      end
+    end
+
+    HTTP_OK_RANGE = 200..299
+    ANNOTATION = "shipit.shopify.io/restart"
+
+    # Initializes the restart task
+    #
+    # @param context [String] Kubernetes context / cluster
+    # @param namespace [String] Kubernetes namespace
+    # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
+    # @param max_watch_seconds [Integer] Timeout in seconds
+    def initialize(context:, namespace:, logger: nil, max_watch_seconds: nil)
+      @logger = logger || Krane::FormattedLogger.build(namespace, context)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @context = context
+      @namespace = namespace
+      @max_watch_seconds = max_watch_seconds
+    end
+
+    # Runs the task, returning a boolean representing success or failure
+    #
+    # @return [Boolean]
+    def run(*args)
+      perform!(*args)
+      true
+    rescue FatalDeploymentError
+      false
+    end
+    alias_method :perform, :run
+
+    # Runs the task, raising exceptions in case of issues
+    #
+    # @param deployments_names [Array<String>] Array of workload names to restart
+    # @param selector [Hash] Selector(s) parsed by Krane::LabelSelector
+    # @param verify_result [Boolean] Wait for completion and verify success
+    #
+    # @return [nil]
+    def run!(deployments_names = nil, selector: nil, verify_result: true)
+      start = Time.now.utc
+      @logger.reset
+
+      @logger.phase_heading("Initializing restart")
+      verify_config!
+      deployments = identify_target_deployments(deployments_names, selector: selector)
+
+      @logger.phase_heading("Triggering restart by touching ENV[RESTARTED_AT]")
+      patch_kubeclient_deployments(deployments)
+
+      if verify_result
+        @logger.phase_heading("Waiting for rollout")
+        resources = build_watchables(deployments, start)
+        verify_restart(resources)
+      else
+        warning = "Result verification is disabled for this task"
+        @logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
+      end
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
+      @logger.print_summary(:success)
+    rescue DeploymentTimeoutError
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
+      @logger.print_summary(:timed_out)
+      raise
+    rescue FatalDeploymentError => error
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
+      @logger.summary.add_action(error.message) if error.message != error.class.to_s
+      @logger.print_summary(:failure)
+      raise
+    end
+    alias_method :perform!, :run!
+
+    private
+
+    def tags(status, deployments)
+      %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
+    end
+
+    def identify_target_deployments(deployment_names, selector: nil)
+      if deployment_names.nil?
+        deployments = if selector.nil?
+          @logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")
+          v1beta1_kubeclient.get_deployments(namespace: @namespace)
+        else
+          selector_string = selector.to_s
+          @logger.info(
+            "Configured to restart all deployments with the `#{ANNOTATION}` annotation and #{selector_string} selector"
+          )
+          v1beta1_kubeclient.get_deployments(namespace: @namespace, label_selector: selector_string)
+        end
+        deployments.select! { |d| d.metadata.annotations[ANNOTATION] }
+
+        if deployments.none?
+          raise FatalRestartError, "no deployments with the `#{ANNOTATION}` annotation found in namespace #{@namespace}"
+        end
+      elsif deployment_names.empty?
+        raise FatalRestartError, "Configured to restart deployments by name, but list of names was blank"
+      elsif !selector.nil?
+        raise FatalRestartError, "Can't specify deployment names and selector at the same time"
+      else
+        deployment_names = deployment_names.uniq
+        list = deployment_names.join(', ')
+        @logger.info("Configured to restart deployments by name: #{list}")
+
+        deployments = fetch_deployments(deployment_names)
+        if deployments.none?
+          raise FatalRestartError, "no deployments with names #{list} found in namespace #{@namespace}"
+        end
+      end
+      deployments
+    end
+
+    def build_watchables(kubeclient_resources, started)
+      kubeclient_resources.map do |d|
+        definition = d.to_h.deep_stringify_keys
+        r = Deployment.new(namespace: @namespace, context: @context, definition: definition, logger: @logger)
+        r.deploy_started_at = started # we don't care what happened to the resource before the restart cmd ran
+        r
+      end
+    end
+
+    def patch_deployment_with_restart(record)
+      v1beta1_kubeclient.patch_deployment(
+        record.metadata.name,
+        build_patch_payload(record),
+        @namespace
+      )
+    end
+
+    def patch_kubeclient_deployments(deployments)
+      deployments.each do |record|
+        begin
+          patch_deployment_with_restart(record)
+          @logger.info("Triggered `#{record.metadata.name}` restart")
+        rescue Kubeclient::HttpError => e
+          raise RestartAPIError.new(record.metadata.name, e.message)
+        end
+      end
+    end
+
+    def fetch_deployments(list)
+      list.map do |name|
+        record = nil
+        begin
+          record = v1beta1_kubeclient.get_deployment(name, @namespace)
+        rescue Kubeclient::ResourceNotFoundError
+          raise FatalRestartError, "Deployment `#{name}` not found in namespace `#{@namespace}`"
+        end
+        record
+      end
+    end
+
+    def build_patch_payload(deployment)
+      containers = deployment.spec.template.spec.containers
+      {
+        spec: {
+          template: {
+            spec: {
+              containers: containers.map do |container|
+                {
+                  name: container.name,
+                  env: [{ name: "RESTARTED_AT", value: Time.now.to_i.to_s }],
+                }
+              end,
+            },
+          },
+        },
+      }
+    end
+
+    def verify_restart(resources)
+      ResourceWatcher.new(resources: resources, operation_name: "restart",
+        timeout: @max_watch_seconds, task_config: @task_config).run
+      failed_resources = resources.reject(&:deploy_succeeded?)
+      success = failed_resources.empty?
+      if !success && failed_resources.all?(&:deploy_timed_out?)
+        raise DeploymentTimeoutError
+      end
+      raise FatalDeploymentError unless success
+    end
+
+    def verify_config!
+      task_config_validator = TaskConfigValidator.new(@task_config, kubectl, kubeclient_builder)
+      unless task_config_validator.valid?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph(task_config_validator.errors.map { |err| "- #{err}" }.join("\n"))
+        raise Krane::TaskConfigurationError
+      end
+    end
+
+    def kubeclient
+      @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
+    end
+
+    def kubectl
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
+    end
+
+    def v1beta1_kubeclient
+      @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
+    end
+
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new
+    end
+  end
+end