tobsch-krane 1.0.0

data/lib/krane/resource_deployer.rb

@@ -0,0 +1,265 @@
+# frozen_string_literal: true
+
+require 'krane/resource_watcher'
+require 'krane/concerns/template_reporting'
+
+module Krane
+  class ResourceDeployer
+    extend Krane::StatsD::MeasureMethods
+    include Krane::TemplateReporting
+
+    delegate :logger, to: :@task_config
+    attr_reader :statsd_tags
+
+    def initialize(task_config:, prune_whitelist:, global_timeout:, current_sha: nil, selector:, statsd_tags:)
+      @task_config = task_config
+      @prune_whitelist = prune_whitelist
+      @global_timeout = global_timeout
+      @current_sha = current_sha
+      @selector = selector
+      @statsd_tags = statsd_tags
+    end
+
+    def deploy!(resources, verify_result, prune)
+      if verify_result
+        deploy_all_resources(resources, prune: prune, verify: true)
+        failed_resources = resources.reject(&:deploy_succeeded?)
+        success = failed_resources.empty?
+        if !success && failed_resources.all?(&:deploy_timed_out?)
+          raise DeploymentTimeoutError
+        end
+        raise FatalDeploymentError unless success
+      else
+        deploy_all_resources(resources, prune: prune, verify: false)
+        logger.summary.add_action("deployed #{resources.length} #{'resource'.pluralize(resources.length)}")
+        warning = <<~MSG
+          Deploy result verification is disabled for this deploy.
+          This means the desired changes were communicated to Kubernetes, but the deploy did not make sure they actually succeeded.
+        MSG
+        logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
+      end
+    end
+
+    def predeploy_priority_resources(resource_list, predeploy_sequence)
+      bare_pods = resource_list.select { |resource| resource.is_a?(Pod) }
+      if bare_pods.count == 1
+        bare_pods.first.stream_logs = true
+      end
+
+      predeploy_sequence.each do |resource_type|
+        matching_resources = resource_list.select { |r| r.type == resource_type }
+        next if matching_resources.empty?
+        deploy_resources(matching_resources, verify: true, record_summary: false)
+
+        failed_resources = matching_resources.reject(&:deploy_succeeded?)
+        fail_count = failed_resources.length
+        if fail_count > 0
+          Krane::Concurrency.split_across_threads(failed_resources) do |r|
+            r.sync_debug_info(kubectl)
+          end
+          failed_resources.each { |r| logger.summary.add_paragraph(r.debug_message) }
+          raise FatalDeploymentError, "Failed to deploy #{fail_count} priority #{'resource'.pluralize(fail_count)}"
+        end
+        logger.blank_line
+      end
+    end
+    measure_method(:predeploy_priority_resources, 'priority_resources.duration')
+
+    private
+
+    def deploy_all_resources(resources, prune: false, verify:, record_summary: true)
+      deploy_resources(resources, prune: prune, verify: verify, record_summary: record_summary)
+    end
+    measure_method(:deploy_all_resources, 'normal_resources.duration')
+
+    def deploy_resources(resources, prune: false, verify:, record_summary: true)
+      return if resources.empty?
+      deploy_started_at = Time.now.utc
+
+      if resources.length > 1
+        logger.info("Deploying resources:")
+        resources.each do |r|
+          logger.info("- #{r.id} (#{r.pretty_timeout_type})")
+        end
+      else
+        resource = resources.first
+        logger.info("Deploying #{resource.id} (#{resource.pretty_timeout_type})")
+      end
+
+      # Apply can be done in one large batch, the rest have to be done individually
+      applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
+      # Prunable resources should also applied so that they can  be pruned
+      pruneable_types = @prune_whitelist.map { |t| t.split("/").last }
+      applyables += individuals.select { |r| pruneable_types.include?(r.type) }
+
+      individuals.each do |individual_resource|
+        individual_resource.deploy_started_at = Time.now.utc
+
+        case individual_resource.deploy_method
+        when :create
+          err, status = create_resource(individual_resource)
+        when :replace
+          err, status = replace_or_create_resource(individual_resource)
+        when :replace_force
+          err, status = replace_or_create_resource(individual_resource, force: true)
+        else
+          # Fail Fast! This is a programmer mistake.
+          raise ArgumentError, "Unexpected deploy method! (#{individual_resource.deploy_method.inspect})"
+        end
+
+        next if status.success?
+
+        raise FatalDeploymentError, <<~MSG
+          Failed to replace or create resource: #{individual_resource.id}
+          #{individual_resource.sensitive_template_content? ? '<suppressed sensitive output>' : err}
+        MSG
+      end
+
+      apply_all(applyables, prune)
+
+      if verify
+        watcher = Krane::ResourceWatcher.new(resources: resources, deploy_started_at: deploy_started_at,
+          timeout: @global_timeout, task_config: @task_config, sha: @current_sha)
+        watcher.run(record_summary: record_summary)
+      end
+    end
+
+    def apply_all(resources, prune)
+      return unless resources.present?
+      command = %w(apply)
+
+      Dir.mktmpdir do |tmp_dir|
+        resources.each do |r|
+          FileUtils.symlink(r.file_path, tmp_dir)
+          r.deploy_started_at = Time.now.utc
+        end
+        command.push("-f", tmp_dir)
+
+        if prune && @prune_whitelist.present?
+          command.push("--prune")
+          if @selector
+            command.push("--selector", @selector.to_s)
+          else
+            command.push("--all")
+          end
+          @prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
+        end
+
+        output_is_sensitive = resources.any?(&:sensitive_template_content?)
+        global_mode = resources.all?(&:global?)
+        out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive,
+          use_namespace: !global_mode)
+
+        if st.success?
+          log_pruning(out) if prune
+        else
+          record_apply_failure(err, resources: resources)
+          raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
+        end
+      end
+    end
+    measure_method(:apply_all)
+
+    def log_pruning(kubectl_output)
+      pruned = kubectl_output.scan(/^(.*) pruned$/)
+      return unless pruned.present?
+
+      logger.info("The following resources were pruned: #{pruned.join(', ')}")
+      logger.summary.add_action("pruned #{pruned.length} #{'resource'.pluralize(pruned.length)}")
+    end
+
+    def record_apply_failure(err, resources: [])
+      warn_msg = "WARNING: Any resources not mentioned in the error(s) below were likely created/updated. " \
+        "You may wish to roll back this deploy."
+      logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
+
+      unidentified_errors = []
+      filenames_with_sensitive_content = resources
+        .select(&:sensitive_template_content?)
+        .map { |r| File.basename(r.file_path) }
+
+      server_dry_run_validated_resource = resources
+        .select(&:server_dry_run_validated?)
+        .map { |r| File.basename(r.file_path) }
+
+      err.each_line do |line|
+        bad_files = find_bad_files_from_kubectl_output(line)
+        unless bad_files.present?
+          unidentified_errors << line
+          next
+        end
+
+        bad_files.each do |f|
+          err_msg = f[:err]
+          if filenames_with_sensitive_content.include?(f[:filename])
+            # Hide the error and template contents in case it has sensitive information
+            # we display full error messages as we assume there's no sensitive info leak after server-dry-run
+            err_msg = "SUPPRESSED FOR SECURITY" unless server_dry_run_validated_resource.include?(f[:filename])
+            record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: nil)
+          else
+            record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: f[:content])
+          end
+        end
+      end
+      return unless unidentified_errors.any?
+
+      if (filenames_with_sensitive_content - server_dry_run_validated_resource).present?
+        warn_msg = "WARNING: There was an error applying some or all resources. The raw output may be sensitive and " \
+          "so cannot be displayed."
+        logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
+      else
+        heading = ColorizedString.new('Unidentified error(s):').red
+        msg = FormattedLogger.indent_four(unidentified_errors.join)
+        logger.summary.add_paragraph("#{heading}\n#{msg}")
+      end
+    end
+
+    def replace_or_create_resource(resource, force: false)
+      args = if force
+        ["replace", "--force", "--cascade", "-f", resource.file_path]
+      else
+        ["replace", "-f", resource.file_path]
+      end
+
+      _, err, status = kubectl.run(*args, log_failure: false, output_is_sensitive: resource.sensitive_template_content?,
+        raise_if_not_found: true, use_namespace: !resource.global?)
+
+      [err, status]
+    rescue Krane::Kubectl::ResourceNotFoundError
+      # it doesn't exist so we can't replace it, we try to create it
+      create_resource(resource)
+    end
+
+    def create_resource(resource)
+      out, err, status = kubectl.run("create", "-f", resource.file_path, log_failure: false,
+        output: 'json', output_is_sensitive: resource.sensitive_template_content?,
+        use_namespace: !resource.global?)
+
+      # For resources that rely on a generateName attribute, we get the `name` from the result of the call to `create`
+      # We must explicitly set this name value so that the `apply` step for pruning can run successfully
+      if status.success? && resource.uses_generate_name?
+        resource.use_generated_name(JSON.parse(out))
+      end
+
+      [err, status]
+    end
+
+    # Inspect the file referenced in the kubectl stderr
+    # to make it easier for developer to understand what's going on
+    def find_bad_files_from_kubectl_output(line)
+      # stderr often contains one or more lines like the following, from which we can extract the file path(s):
+      # Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
+
+      line.scan(%r{"(/\S+\.ya?ml\S*)"}).each_with_object([]) do |matches, bad_files|
+        matches.each do |path|
+          content = File.read(path) if File.file?(path)
+          bad_files << { filename: File.basename(path), err: line, content: content }
+        end
+      end
+    end
+
+    def kubectl
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
+    end
+  end
+end

data/lib/krane/resource_watcher.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'krane/concurrency'
+require 'krane/resource_cache'
+
+module Krane
+  class ResourceWatcher
+    extend Krane::StatsD::MeasureMethods
+    delegate :namespace, :context, :logger, to: :@task_config
+
+    def initialize(resources:, task_config:, deploy_started_at: Time.now.utc,
+      operation_name: "deploy", timeout: nil, sha: nil)
+      unless resources.is_a?(Enumerable)
+        raise ArgumentError, <<~MSG
+          ResourceWatcher expects Enumerable collection, got `#{resources.class}` instead
+        MSG
+      end
+      @resources = resources
+      @task_config = task_config
+      @deploy_started_at = deploy_started_at
+      @operation_name = operation_name
+      @timeout = timeout
+      @sha = sha
+    end
+
+    def run(delay_sync: 3.seconds, reminder_interval: 30.seconds, record_summary: true)
+      last_message_logged_at = monitoring_started = Time.now.utc
+      remainder = @resources.dup
+
+      while remainder.present?
+        report_and_give_up(remainder) if global_timeout?(monitoring_started)
+        sleep_until_next_sync(delay_sync)
+
+        sync_resources(remainder)
+
+        new_successes, remainder = remainder.partition(&:deploy_succeeded?)
+        new_failures, remainder = remainder.partition(&:deploy_failed?)
+        new_timeouts, remainder = remainder.partition(&:deploy_timed_out?)
+
+        if new_successes.present? || new_failures.present? || new_timeouts.present?
+          report_what_just_happened(new_successes, new_failures, new_timeouts)
+          report_what_is_left(remainder, reminder: false)
+          last_message_logged_at = Time.now.utc
+        elsif due_for_reminder?(last_message_logged_at, reminder_interval)
+          report_what_is_left(remainder, reminder: true)
+          last_message_logged_at = Time.now.utc
+        end
+      end
+      record_statuses_for_summary(@resources) if record_summary
+    end
+
+    private
+
+    def sync_resources(resources)
+      cache = ResourceCache.new(@task_config)
+      Krane::Concurrency.split_across_threads(resources) { |r| r.sync(cache) }
+      resources.each(&:after_sync)
+    end
+    measure_method(:sync_resources, "sync.duration")
+
+    def statsd_tags
+      {
+        namespace: namespace,
+        context: context,
+        sha: @sha,
+      }
+    end
+
+    def global_timeout?(started_at)
+      @timeout && (Time.now.utc - started_at > @timeout)
+    end
+
+    def sleep_until_next_sync(min_interval)
+      @next_sync_time ||= Time.now.utc
+      if (sleep_duration = @next_sync_time - Time.now.utc) > 0
+        sleep(sleep_duration)
+      end
+      @next_sync_time = Time.now.utc + min_interval
+    end
+
+    def report_what_just_happened(new_successes, new_failures, new_timeouts)
+      watch_time = (Time.now.utc - @deploy_started_at).round(1)
+      new_failures.each do |resource|
+        resource.report_status_to_statsd(watch_time)
+        logger.error("#{resource.id} failed to #{@operation_name} after #{watch_time}s")
+      end
+
+      new_timeouts.each do |resource|
+        resource.report_status_to_statsd(watch_time)
+        logger.error("#{resource.id} rollout timed out after #{watch_time}s")
+      end
+
+      if new_successes.present?
+        new_successes.each { |r| r.report_status_to_statsd(watch_time) }
+        success_string = ColorizedString.new("Successfully #{past_tense_operation} in #{watch_time}s:").green
+        logger.info("#{success_string} #{new_successes.map(&:id).join(', ')}")
+      end
+    end
+
+    def report_what_is_left(resources, reminder:)
+      return unless resources.present?
+      resource_list = resources.map(&:id).join(', ')
+      msg = reminder ? "Still waiting for: #{resource_list}" : "Continuing to wait for: #{resource_list}"
+      logger.info(msg)
+    end
+
+    def report_and_give_up(remaining_resources)
+      successful_resources, failed_resources = (@resources - remaining_resources).partition(&:deploy_succeeded?)
+      record_success_statuses(successful_resources)
+      record_failed_statuses(failed_resources, remaining_resources)
+
+      if failed_resources.present? && !failed_resources.all?(&:deploy_timed_out?)
+        raise FatalDeploymentError
+      else
+        raise DeploymentTimeoutError
+      end
+    end
+
+    def record_statuses_for_summary(resources)
+      successful_resources, failed_resources = resources.partition(&:deploy_succeeded?)
+      record_success_statuses(successful_resources)
+      record_failed_statuses(failed_resources)
+    end
+
+    def record_failed_statuses(failed_resources, global_timeouts = [])
+      fail_count = failed_resources.length + global_timeouts.length
+
+      if fail_count > 0
+        timeouts, failures = failed_resources.partition(&:deploy_timed_out?)
+        timeouts += global_timeouts
+        if timeouts.present?
+          logger.summary.add_action(
+            "timed out waiting for #{timeouts.length} #{'resource'.pluralize(timeouts.length)} to #{@operation_name}"
+          )
+        end
+
+        if failures.present?
+          logger.summary.add_action(
+            "failed to #{@operation_name} #{failures.length} #{'resource'.pluralize(failures.length)}"
+          )
+        end
+
+        kubectl = Kubectl.new(task_config: @task_config, log_failure_by_default: false)
+        Krane::Concurrency.split_across_threads(failed_resources + global_timeouts) do |r|
+          r.sync_debug_info(kubectl)
+        end
+
+        failed_resources.each { |r| logger.summary.add_paragraph(r.debug_message) }
+        global_timeouts.each { |r| logger.summary.add_paragraph(r.debug_message(:gave_up, timeout: @timeout)) }
+      end
+    end
+
+    def record_success_statuses(successful_resources)
+      success_count = successful_resources.length
+      if success_count > 0
+        logger.summary.add_action("successfully #{past_tense_operation} #{success_count} "\
+          "#{'resource'.pluralize(success_count)}")
+        final_statuses = successful_resources.map(&:pretty_status).join("\n")
+        logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
+      end
+    end
+
+    def due_for_reminder?(last_message_logged_at, reminder_interval)
+      (last_message_logged_at.to_f + reminder_interval.to_f) <= Time.now.utc.to_f
+    end
+
+    def past_tense_operation
+      @operation_name == "run" ? "ran" : "#{@operation_name}ed"
+    end
+  end
+end

data/lib/krane/restart_task.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+require 'krane/common'
+require 'krane/kubernetes_resource'
+require 'krane/kubernetes_resource/deployment'
+require 'krane/kubeclient_builder'
+require 'krane/resource_watcher'
+require 'krane/kubectl'
+
+module Krane
+  # Restart the pods in one or more deployments
+  class RestartTask
+    class FatalRestartError < FatalDeploymentError; end
+
+    class RestartAPIError < FatalRestartError
+      def initialize(deployment_name, response)
+        super("Failed to restart #{deployment_name}. " \
+            "API returned non-200 response code (#{response.code})\n" \
+            "Response:\n#{response.body}")
+      end
+    end
+
+    HTTP_OK_RANGE = 200..299
+    ANNOTATION = "shipit.shopify.io/restart"
+
+    # Initializes the restart task
+    #
+    # @param context [String] Kubernetes context / cluster (*required*)
+    # @param namespace [String] Kubernetes namespace (*required*)
+    # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
+    # @param global_timeout [Integer] Timeout in seconds
+    def initialize(context:, namespace:, logger: nil, global_timeout: nil)
+      @logger = logger || Krane::FormattedLogger.build(namespace, context)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @context = context
+      @namespace = namespace
+      @global_timeout = global_timeout
+    end
+
+    # Runs the task, returning a boolean representing success or failure
+    #
+    # @return [Boolean]
+    def run(*args)
+      perform!(*args)
+      true
+    rescue FatalDeploymentError
+      false
+    end
+    alias_method :perform, :run
+
+    # Runs the task, raising exceptions in case of issues
+    #
+    # @param deployments [Array<String>] Array of workload names to restart
+    # @param selector [Hash] Selector(s) parsed by Krane::LabelSelector
+    # @param verify_result [Boolean] Wait for completion and verify success
+    #
+    # @return [nil]
+    def run!(deployments: nil, selector: nil, verify_result: true)
+      start = Time.now.utc
+      @logger.reset
+
+      @logger.phase_heading("Initializing restart")
+      verify_config!
+      deployments = identify_target_deployments(deployments, selector: selector)
+
+      @logger.phase_heading("Triggering restart by touching ENV[RESTARTED_AT]")
+      patch_kubeclient_deployments(deployments)
+
+      if verify_result
+        @logger.phase_heading("Waiting for rollout")
+        resources = build_watchables(deployments, start)
+        verify_restart(resources)
+      else
+        warning = "Result verification is disabled for this task"
+        @logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
+      end
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
+      @logger.print_summary(:success)
+    rescue DeploymentTimeoutError
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
+      @logger.print_summary(:timed_out)
+      raise
+    rescue FatalDeploymentError => error
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
+      @logger.summary.add_action(error.message) if error.message != error.class.to_s
+      @logger.print_summary(:failure)
+      raise
+    end
+    alias_method :perform!, :run!
+
+    private
+
+    def tags(status, deployments)
+      %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
+    end
+
+    def identify_target_deployments(deployment_names, selector: nil)
+      if deployment_names.nil?
+        deployments = if selector.nil?
+          @logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")
+          apps_v1_kubeclient.get_deployments(namespace: @namespace)
+        else
+          selector_string = selector.to_s
+          @logger.info(
+            "Configured to restart all deployments with the `#{ANNOTATION}` annotation and #{selector_string} selector"
+          )
+          apps_v1_kubeclient.get_deployments(namespace: @namespace, label_selector: selector_string)
+        end
+        deployments.select! { |d| d.metadata.annotations[ANNOTATION] }
+
+        if deployments.none?
+          raise FatalRestartError, "no deployments with the `#{ANNOTATION}` annotation found in namespace #{@namespace}"
+        end
+      elsif deployment_names.empty?
+        raise FatalRestartError, "Configured to restart deployments by name, but list of names was blank"
+      elsif !selector.nil?
+        raise FatalRestartError, "Can't specify deployment names and selector at the same time"
+      else
+        deployment_names = deployment_names.uniq
+        list = deployment_names.join(', ')
+        @logger.info("Configured to restart deployments by name: #{list}")
+
+        deployments = fetch_deployments(deployment_names)
+        if deployments.none?
+          raise FatalRestartError, "no deployments with names #{list} found in namespace #{@namespace}"
+        end
+      end
+      deployments
+    end
+
+    def build_watchables(kubeclient_resources, started)
+      kubeclient_resources.map do |d|
+        definition = d.to_h.deep_stringify_keys
+        r = Deployment.new(namespace: @namespace, context: @context, definition: definition, logger: @logger)
+        r.deploy_started_at = started # we don't care what happened to the resource before the restart cmd ran
+        r
+      end
+    end
+
+    def patch_deployment_with_restart(record)
+      apps_v1_kubeclient.patch_deployment(
+        record.metadata.name,
+        build_patch_payload(record),
+        @namespace
+      )
+    end
+
+    def patch_kubeclient_deployments(deployments)
+      deployments.each do |record|
+        begin
+          patch_deployment_with_restart(record)
+          @logger.info("Triggered `#{record.metadata.name}` restart")
+        rescue Kubeclient::HttpError => e
+          raise RestartAPIError.new(record.metadata.name, e.message)
+        end
+      end
+    end
+
+    def fetch_deployments(list)
+      list.map do |name|
+        record = nil
+        begin
+          record = apps_v1_kubeclient.get_deployment(name, @namespace)
+        rescue Kubeclient::ResourceNotFoundError
+          raise FatalRestartError, "Deployment `#{name}` not found in namespace `#{@namespace}`"
+        end
+        record
+      end
+    end
+
+    def build_patch_payload(deployment)
+      containers = deployment.spec.template.spec.containers
+      {
+        spec: {
+          template: {
+            spec: {
+              containers: containers.map do |container|
+                {
+                  name: container.name,
+                  env: [{ name: "RESTARTED_AT", value: Time.now.to_i.to_s }],
+                }
+              end,
+            },
+          },
+        },
+      }
+    end
+
+    def verify_restart(resources)
+      ResourceWatcher.new(resources: resources, operation_name: "restart",
+        timeout: @global_timeout, task_config: @task_config).run
+      failed_resources = resources.reject(&:deploy_succeeded?)
+      success = failed_resources.empty?
+      if !success && failed_resources.all?(&:deploy_timed_out?)
+        raise DeploymentTimeoutError
+      end
+      raise FatalDeploymentError unless success
+    end
+
+    def verify_config!
+      task_config_validator = TaskConfigValidator.new(@task_config, kubectl, kubeclient_builder)
+      unless task_config_validator.valid?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph(task_config_validator.errors.map { |err| "- #{err}" }.join("\n"))
+        raise Krane::TaskConfigurationError
+      end
+    end
+
+    def apps_v1_kubeclient
+      @apps_v1_kubeclient ||= kubeclient_builder.build_apps_v1_kubeclient(@context)
+    end
+
+    def kubeclient
+      @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
+    end
+
+    def kubectl
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
+    end
+
+    def v1beta1_kubeclient
+      @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
+    end
+
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new
+    end
+  end
+end