kubernetes-deploy 0.29.0 → 1.0.0.pre.2

data/lib/{kubernetes-deploy → krane}/resource_watcher.rb

@@ -1,23 +1,22 @@
 # frozen_string_literal: true
 
-require 'kubernetes-deploy/concurrency'
-require 'kubernetes-deploy/resource_cache'
+require 'krane/concurrency'
+require 'krane/resource_cache'
 
-module KubernetesDeploy
+module Krane
   class ResourceWatcher
-    extend KubernetesDeploy::StatsD::MeasureMethods
+    extend Krane::StatsD::MeasureMethods
+    delegate :namespace, :context, :logger, to: :@task_config
 
-    def initialize(resources:, logger:, context:, namespace:,
-      deploy_started_at: Time.now.utc, operation_name: "deploy", timeout: nil, sha: nil)
+    def initialize(resources:, task_config:, deploy_started_at: Time.now.utc,
+      operation_name: "deploy", timeout: nil, sha: nil)
       unless resources.is_a?(Enumerable)
         raise ArgumentError, <<~MSG
           ResourceWatcher expects Enumerable collection, got `#{resources.class}` instead
         MSG
       end
       @resources = resources
-      @logger = logger
-      @namespace = namespace
-      @context = context
+      @task_config = task_config
       @deploy_started_at = deploy_started_at
       @operation_name = operation_name
       @timeout = timeout
@@ -53,16 +52,16 @@ module KubernetesDeploy
     private
 
     def sync_resources(resources)
-      cache = ResourceCache.new(@namespace, @context, @logger)
-      KubernetesDeploy::Concurrency.split_across_threads(resources) { |r| r.sync(cache) }
+      cache = ResourceCache.new(@task_config)
+      Krane::Concurrency.split_across_threads(resources) { |r| r.sync(cache) }
       resources.each(&:after_sync)
     end
     measure_method(:sync_resources, "sync.duration")
 
     def statsd_tags
       {
-        namespace: @namespace,
-        context: @context,
+        namespace: namespace,
+        context: context,
         sha: @sha,
       }
     end
@@ -83,18 +82,18 @@ module KubernetesDeploy
       watch_time = (Time.now.utc - @deploy_started_at).round(1)
       new_failures.each do |resource|
         resource.report_status_to_statsd(watch_time)
-        @logger.error("#{resource.id} failed to #{@operation_name} after #{watch_time}s")
+        logger.error("#{resource.id} failed to #{@operation_name} after #{watch_time}s")
       end
 
       new_timeouts.each do |resource|
         resource.report_status_to_statsd(watch_time)
-        @logger.error("#{resource.id} rollout timed out after #{watch_time}s")
+        logger.error("#{resource.id} rollout timed out after #{watch_time}s")
       end
 
       if new_successes.present?
         new_successes.each { |r| r.report_status_to_statsd(watch_time) }
         success_string = ColorizedString.new("Successfully #{past_tense_operation} in #{watch_time}s:").green
-        @logger.info("#{success_string} #{new_successes.map(&:id).join(', ')}")
+        logger.info("#{success_string} #{new_successes.map(&:id).join(', ')}")
       end
     end
 
@@ -102,7 +101,7 @@ module KubernetesDeploy
       return unless resources.present?
       resource_list = resources.map(&:id).join(', ')
       msg = reminder ? "Still waiting for: #{resource_list}" : "Continuing to wait for: #{resource_list}"
-      @logger.info(msg)
+      logger.info(msg)
     end
 
     def report_and_give_up(remaining_resources)
@@ -130,34 +129,34 @@ module KubernetesDeploy
         timeouts, failures = failed_resources.partition(&:deploy_timed_out?)
         timeouts += global_timeouts
         if timeouts.present?
-          @logger.summary.add_action(
+          logger.summary.add_action(
             "timed out waiting for #{timeouts.length} #{'resource'.pluralize(timeouts.length)} to #{@operation_name}"
           )
         end
 
         if failures.present?
-          @logger.summary.add_action(
+          logger.summary.add_action(
             "failed to #{@operation_name} #{failures.length} #{'resource'.pluralize(failures.length)}"
           )
         end
 
-        kubectl = Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: false)
-        KubernetesDeploy::Concurrency.split_across_threads(failed_resources + global_timeouts) do |r|
+        kubectl = Kubectl.new(task_config: @task_config, log_failure_by_default: false)
+        Krane::Concurrency.split_across_threads(failed_resources + global_timeouts) do |r|
           r.sync_debug_info(kubectl)
         end
 
-        failed_resources.each { |r| @logger.summary.add_paragraph(r.debug_message) }
-        global_timeouts.each { |r| @logger.summary.add_paragraph(r.debug_message(:gave_up, timeout: @timeout)) }
+        failed_resources.each { |r| logger.summary.add_paragraph(r.debug_message) }
+        global_timeouts.each { |r| logger.summary.add_paragraph(r.debug_message(:gave_up, timeout: @timeout)) }
       end
     end
 
     def record_success_statuses(successful_resources)
       success_count = successful_resources.length
       if success_count > 0
-        @logger.summary.add_action("successfully #{past_tense_operation} #{success_count} "\
+        logger.summary.add_action("successfully #{past_tense_operation} #{success_count} "\
           "#{'resource'.pluralize(success_count)}")
         final_statuses = successful_resources.map(&:pretty_status).join("\n")
-        @logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
+        logger.summary.add_paragraph("#{ColorizedString.new('Successful resources').green}\n#{final_statuses}")
       end
     end
 

data/lib/krane/restart_task.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+require 'krane/common'
+require 'krane/kubernetes_resource'
+require 'krane/kubernetes_resource/deployment'
+require 'krane/kubeclient_builder'
+require 'krane/resource_watcher'
+require 'krane/kubectl'
+
+module Krane
+  # Restart the pods in one or more deployments
+  class RestartTask
+    class FatalRestartError < FatalDeploymentError; end
+
+    class RestartAPIError < FatalRestartError
+      def initialize(deployment_name, response)
+        super("Failed to restart #{deployment_name}. " \
+            "API returned non-200 response code (#{response.code})\n" \
+            "Response:\n#{response.body}")
+      end
+    end
+
+    HTTP_OK_RANGE = 200..299
+    ANNOTATION = "shipit.shopify.io/restart"
+
+    # Initializes the restart task
+    #
+    # @param context [String] Kubernetes context / cluster
+    # @param namespace [String] Kubernetes namespace
+    # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
+    # @param max_watch_seconds [Integer] Timeout in seconds
+    def initialize(context:, namespace:, logger: nil, max_watch_seconds: nil)
+      @logger = logger || Krane::FormattedLogger.build(namespace, context)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @context = context
+      @namespace = namespace
+      @max_watch_seconds = max_watch_seconds
+    end
+
+    # Runs the task, returning a boolean representing success or failure
+    #
+    # @return [Boolean]
+    def run(*args)
+      perform!(*args)
+      true
+    rescue FatalDeploymentError
+      false
+    end
+    alias_method :perform, :run
+
+    # Runs the task, raising exceptions in case of issues
+    #
+    # @param deployments_names [Array<String>] Array of workload names to restart
+    # @param selector [Hash] Selector(s) parsed by Krane::LabelSelector
+    # @param verify_result [Boolean] Wait for completion and verify success
+    #
+    # @return [nil]
+    def run!(deployments_names = nil, selector: nil, verify_result: true)
+      start = Time.now.utc
+      @logger.reset
+
+      @logger.phase_heading("Initializing restart")
+      verify_config!
+      deployments = identify_target_deployments(deployments_names, selector: selector)
+
+      @logger.phase_heading("Triggering restart by touching ENV[RESTARTED_AT]")
+      patch_kubeclient_deployments(deployments)
+
+      if verify_result
+        @logger.phase_heading("Waiting for rollout")
+        resources = build_watchables(deployments, start)
+        verify_restart(resources)
+      else
+        warning = "Result verification is disabled for this task"
+        @logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
+      end
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
+      @logger.print_summary(:success)
+    rescue DeploymentTimeoutError
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
+      @logger.print_summary(:timed_out)
+      raise
+    rescue FatalDeploymentError => error
+      StatsD.client.distribution('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
+      @logger.summary.add_action(error.message) if error.message != error.class.to_s
+      @logger.print_summary(:failure)
+      raise
+    end
+    alias_method :perform!, :run!
+
+    private
+
+    def tags(status, deployments)
+      %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
+    end
+
+    def identify_target_deployments(deployment_names, selector: nil)
+      if deployment_names.nil?
+        deployments = if selector.nil?
+          @logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")
+          apps_v1_kubeclient.get_deployments(namespace: @namespace)
+        else
+          selector_string = selector.to_s
+          @logger.info(
+            "Configured to restart all deployments with the `#{ANNOTATION}` annotation and #{selector_string} selector"
+          )
+          apps_v1_kubeclient.get_deployments(namespace: @namespace, label_selector: selector_string)
+        end
+        deployments.select! { |d| d.metadata.annotations[ANNOTATION] }
+
+        if deployments.none?
+          raise FatalRestartError, "no deployments with the `#{ANNOTATION}` annotation found in namespace #{@namespace}"
+        end
+      elsif deployment_names.empty?
+        raise FatalRestartError, "Configured to restart deployments by name, but list of names was blank"
+      elsif !selector.nil?
+        raise FatalRestartError, "Can't specify deployment names and selector at the same time"
+      else
+        deployment_names = deployment_names.uniq
+        list = deployment_names.join(', ')
+        @logger.info("Configured to restart deployments by name: #{list}")
+
+        deployments = fetch_deployments(deployment_names)
+        if deployments.none?
+          raise FatalRestartError, "no deployments with names #{list} found in namespace #{@namespace}"
+        end
+      end
+      deployments
+    end
+
+    def build_watchables(kubeclient_resources, started)
+      kubeclient_resources.map do |d|
+        definition = d.to_h.deep_stringify_keys
+        r = Deployment.new(namespace: @namespace, context: @context, definition: definition, logger: @logger)
+        r.deploy_started_at = started # we don't care what happened to the resource before the restart cmd ran
+        r
+      end
+    end
+
+    def patch_deployment_with_restart(record)
+      apps_v1_kubeclient.patch_deployment(
+        record.metadata.name,
+        build_patch_payload(record),
+        @namespace
+      )
+    end
+
+    def patch_kubeclient_deployments(deployments)
+      deployments.each do |record|
+        begin
+          patch_deployment_with_restart(record)
+          @logger.info("Triggered `#{record.metadata.name}` restart")
+        rescue Kubeclient::HttpError => e
+          raise RestartAPIError.new(record.metadata.name, e.message)
+        end
+      end
+    end
+
+    def fetch_deployments(list)
+      list.map do |name|
+        record = nil
+        begin
+          record = apps_v1_kubeclient.get_deployment(name, @namespace)
+        rescue Kubeclient::ResourceNotFoundError
+          raise FatalRestartError, "Deployment `#{name}` not found in namespace `#{@namespace}`"
+        end
+        record
+      end
+    end
+
+    def build_patch_payload(deployment)
+      containers = deployment.spec.template.spec.containers
+      {
+        spec: {
+          template: {
+            spec: {
+              containers: containers.map do |container|
+                {
+                  name: container.name,
+                  env: [{ name: "RESTARTED_AT", value: Time.now.to_i.to_s }],
+                }
+              end,
+            },
+          },
+        },
+      }
+    end
+
+    def verify_restart(resources)
+      ResourceWatcher.new(resources: resources, operation_name: "restart",
+        timeout: @max_watch_seconds, task_config: @task_config).run
+      failed_resources = resources.reject(&:deploy_succeeded?)
+      success = failed_resources.empty?
+      if !success && failed_resources.all?(&:deploy_timed_out?)
+        raise DeploymentTimeoutError
+      end
+      raise FatalDeploymentError unless success
+    end
+
+    def verify_config!
+      task_config_validator = TaskConfigValidator.new(@task_config, kubectl, kubeclient_builder)
+      unless task_config_validator.valid?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph(task_config_validator.errors.map { |err| "- #{err}" }.join("\n"))
+        raise Krane::TaskConfigurationError
+      end
+    end
+
+    def apps_v1_kubeclient
+      @apps_v1_kubeclient ||= kubeclient_builder.build_apps_v1_kubeclient(@context)
+    end
+
+    def kubeclient
+      @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
+    end
+
+    def kubectl
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
+    end
+
+    def v1beta1_kubeclient
+      @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
+    end
+
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new
+    end
+  end
+end

data/lib/{kubernetes-deploy → krane}/rollout_conditions.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-module KubernetesDeploy
+module Krane
   class RolloutConditionsError < StandardError
   end
 

data/lib/krane/runner_task.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+require 'tempfile'
+
+require 'krane/common'
+require 'krane/kubeclient_builder'
+require 'krane/kubectl'
+require 'krane/resource_cache'
+require 'krane/resource_watcher'
+require 'krane/kubernetes_resource'
+require 'krane/kubernetes_resource/pod'
+require 'krane/runner_task_config_validator'
+
+module Krane
+  # Run a pod that exits upon completing a task
+  class RunnerTask
+    class TaskTemplateMissingError < TaskConfigurationError; end
+
+    attr_reader :pod_name
+
+    # Initializes the runner task
+    #
+    # @param namespace [String] Kubernetes namespace
+    # @param context [String] Kubernetes context / cluster
+    # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
+    # @param max_watch_seconds [Integer] Timeout in seconds
+    def initialize(namespace:, context:, logger: nil, max_watch_seconds: nil)
+      @logger = logger || Krane::FormattedLogger.build(namespace, context)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @namespace = namespace
+      @context = context
+      @max_watch_seconds = max_watch_seconds
+    end
+
+    # Runs the task, returning a boolean representing success or failure
+    #
+    # @return [Boolean]
+    def run(*args)
+      run!(*args)
+      true
+    rescue DeploymentTimeoutError, FatalDeploymentError
+      false
+    end
+
+    # Runs the task, raising exceptions in case of issues
+    #
+    # @param task_template [String] The template file you'll be rendering
+    # @param entrypoint [Array<String>] Override the default command in the container image
+    # @param args [Array<String>] Override the default arguments for the command
+    # @param env_vars [Array<String>] List of env vars
+    # @param verify_result [Boolean] Wait for completion and verify pod success
+    #
+    # @return [nil]
+    def run!(task_template:, entrypoint:, args:, env_vars: [], verify_result: true)
+      start = Time.now.utc
+      @logger.reset
+
+      @logger.phase_heading("Initializing task")
+
+      @logger.info("Validating configuration")
+      verify_config!(task_template, args)
+      @logger.info("Using namespace '#{@namespace}' in context '#{@context}'")
+
+      pod = build_pod(task_template, entrypoint, args, env_vars, verify_result)
+      validate_pod(pod)
+
+      @logger.phase_heading("Running pod")
+      create_pod(pod)
+
+      if verify_result
+        @logger.phase_heading("Streaming logs")
+        watch_pod(pod)
+      else
+        record_status_once(pod)
+      end
+      StatsD.client.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('success'))
+      @logger.print_summary(:success)
+    rescue DeploymentTimeoutError
+      StatsD.client.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('timeout'))
+      @logger.print_summary(:timed_out)
+      raise
+    rescue FatalDeploymentError
+      StatsD.client.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('failure'))
+      @logger.print_summary(:failure)
+      raise
+    end
+
+    private
+
+    def create_pod(pod)
+      @logger.info("Creating pod '#{pod.name}'")
+      pod.deploy_started_at = Time.now.utc
+      kubeclient.create_pod(pod.to_kubeclient_resource)
+      @pod_name = pod.name
+      @logger.info("Pod creation succeeded")
+    rescue Kubeclient::HttpError => e
+      msg = "Failed to create pod: #{e.class.name}: #{e.message}"
+      @logger.summary.add_paragraph(msg)
+      raise FatalDeploymentError, msg
+    end
+
+    def build_pod(template_name, entrypoint, args, env_vars, verify_result)
+      task_template = get_template(template_name)
+      @logger.info("Using template '#{template_name}'")
+      pod_template = build_pod_definition(task_template)
+      set_container_overrides!(pod_template, entrypoint, args, env_vars)
+      ensure_valid_restart_policy!(pod_template, verify_result)
+      Pod.new(namespace: @namespace, context: @context, logger: @logger, stream_logs: true,
+                    definition: pod_template.to_hash.deep_stringify_keys, statsd_tags: [])
+    end
+
+    def validate_pod(pod)
+      pod.validate_definition(kubectl)
+    end
+
+    def watch_pod(pod)
+      rw = ResourceWatcher.new(resources: [pod], timeout: @max_watch_seconds,
+        operation_name: "run", task_config: @task_config)
+      rw.run(delay_sync: 1, reminder_interval: 30.seconds)
+      raise DeploymentTimeoutError if pod.deploy_timed_out?
+      raise FatalDeploymentError if pod.deploy_failed?
+    end
+
+    def record_status_once(pod)
+      cache = ResourceCache.new(@task_config)
+      pod.sync(cache)
+      warning = <<~STRING
+        #{ColorizedString.new('Result verification is disabled for this task.').yellow}
+        The following status was observed immediately after pod creation:
+        #{pod.pretty_status}
+      STRING
+      @logger.summary.add_paragraph(warning)
+    end
+
+    def verify_config!(task_template, args)
+      task_config_validator = RunnerTaskConfigValidator.new(task_template, args, @task_config, kubectl,
+        kubeclient_builder)
+      unless task_config_validator.valid?
+        @logger.summary.add_action("Configuration invalid")
+        @logger.summary.add_paragraph([task_config_validator.errors].map { |err| "- #{err}" }.join("\n"))
+        raise Krane::TaskConfigurationError
+      end
+    end
+
+    def get_template(template_name)
+      pod_template = kubeclient.get_pod_template(template_name, @namespace)
+      pod_template.template
+    rescue Kubeclient::ResourceNotFoundError
+      msg = "Pod template `#{template_name}` not found in namespace `#{@namespace}`, context `#{@context}`"
+      @logger.summary.add_paragraph(msg)
+      raise TaskTemplateMissingError, msg
+    rescue Kubeclient::HttpError => error
+      raise FatalKubeAPIError, "Error retrieving pod template: #{error.class.name}: #{error.message}"
+    end
+
+    def build_pod_definition(base_template)
+      pod_definition = base_template.dup
+      pod_definition.kind = 'Pod'
+      pod_definition.apiVersion = 'v1'
+      pod_definition.metadata.namespace = @namespace
+
+      unique_name = pod_definition.metadata.name + "-" + SecureRandom.hex(8)
+      @logger.warn("Name is too long, using '#{unique_name[0..62]}'") if unique_name.length > 63
+      pod_definition.metadata.name = unique_name[0..62]
+
+      pod_definition
+    end
+
+    def set_container_overrides!(pod_definition, entrypoint, args, env_vars)
+      container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
+      if container.nil?
+        message = "Pod spec does not contain a template container called 'task-runner'"
+        @logger.summary.add_paragraph(message)
+        raise TaskConfigurationError, message
+      end
+
+      container.command = entrypoint if entrypoint
+      container.args = args if args
+
+      env_args = env_vars.map do |env|
+        key, value = env.split('=', 2)
+        { name: key, value: value }
+      end
+      container.env ||= []
+      container.env = container.env.map(&:to_h) + env_args
+    end
+
+    def ensure_valid_restart_policy!(template, verify)
+      restart_policy = template.spec.restartPolicy
+      if verify && restart_policy != "Never"
+        @logger.warn("Changed Pod RestartPolicy from '#{restart_policy}' to 'Never'. Disable "\
+          "result verification to use '#{restart_policy}'.")
+        template.spec.restartPolicy = "Never"
+      end
+    end
+
+    def kubectl
+      @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
+    end
+
+    def kubeclient
+      @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
+    end
+
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new
+    end
+
+    def statsd_tags(status)
+      %W(namespace:#{@namespace} context:#{@context} status:#{status})
+    end
+  end
+end