kubernetes-deploy 0.29.0 → 1.0.0.pre.2

data/lib/kubernetes-deploy/errors.rb

@@ -1,33 +1,8 @@
 # frozen_string_literal: true
+
 module KubernetesDeploy
   class FatalDeploymentError < StandardError; end
   class FatalKubeAPIError < FatalDeploymentError; end
   class KubectlError < StandardError; end
-  class TaskConfigurationError < FatalDeploymentError; end
-
-  class InvalidTemplateError < FatalDeploymentError
-    attr_reader :content
-    attr_accessor :filename
-    def initialize(err, filename: nil, content: nil)
-      @filename = filename
-      @content = content
-      super(err)
-    end
-  end
-
-  class NamespaceNotFoundError < FatalDeploymentError
-    def initialize(name, context)
-      super("Namespace `#{name}` not found in context `#{context}`")
-    end
-  end
-
   class DeploymentTimeoutError < FatalDeploymentError; end
-
-  class EjsonPrunableError < FatalDeploymentError
-    def initialize
-      super("Found #{KubernetesResource::LAST_APPLIED_ANNOTATION} annotation on " \
-          "#{EjsonSecretProvisioner::EJSON_KEYS_SECRET} secret. " \
-          "kubernetes-deploy will not continue since it is extremely unlikely that this secret should be pruned.")
-    end
-  end
 end

data/lib/kubernetes-deploy/render_task.rb

@@ -1,132 +1,15 @@
 # frozen_string_literal: true
-require 'tempfile'
-
-require 'kubernetes-deploy/common'
-require 'kubernetes-deploy/renderer'
-require 'kubernetes-deploy/template_sets'
+require 'krane/render_task'
+require 'kubernetes-deploy/rescue_krane_exceptions'
 
 module KubernetesDeploy
-  class RenderTask
-    def initialize(logger: nil, current_sha:, template_dir: nil, template_paths: [], bindings:)
-      @logger = logger || KubernetesDeploy::FormattedLogger.build
-      @template_dir = template_dir
-      @template_paths = template_paths.map { |path| File.expand_path(path) }
-      @bindings = bindings
-      @current_sha = current_sha
-    end
+  class RenderTask < ::Krane::RenderTask
+    include RescueKraneExceptions
 
     def run(*args)
-      run!(*args)
-      true
+      super(*args)
     rescue KubernetesDeploy::FatalDeploymentError
       false
     end
-
-    def run!(stream, only_filenames = [])
-      @logger.reset
-      @logger.phase_heading("Initializing render task")
-
-      ts = TemplateSets.from_dirs_and_files(paths: template_sets_paths(only_filenames), logger: @logger)
-
-      validate_configuration(ts, only_filenames)
-      count = render_templates(stream, ts)
-
-      @logger.summary.add_action("Successfully rendered #{count} template(s)")
-      @logger.print_summary(:success)
-    rescue KubernetesDeploy::FatalDeploymentError
-      @logger.print_summary(:failure)
-      raise
-    end
-
-    private
-
-    def template_sets_paths(only_filenames)
-      if @template_paths.present?
-        # Validation will catch @template_paths & @template_dir being present
-        @template_paths
-      elsif only_filenames.blank?
-        [File.expand_path(@template_dir || '')]
-      else
-        absolute_template_dir = File.expand_path(@template_dir || '')
-        only_filenames.map do |name|
-          File.join(absolute_template_dir, name)
-        end
-      end
-    end
-
-    def render_templates(stream, template_sets)
-      @logger.phase_heading("Rendering template(s)")
-      count = 0
-      template_sets.with_resource_definitions_and_filename(render_erb: true,
-          current_sha: @current_sha, bindings: @bindings, raw: true) do |rendered_content, filename|
-        write_to_stream(rendered_content, filename, stream)
-        count += 1
-      end
-
-      count
-    rescue KubernetesDeploy::InvalidTemplateError => exception
-      log_invalid_template(exception)
-      raise
-    end
-
-    def write_to_stream(rendered_content, filename, stream)
-      file_basename = File.basename(filename)
-      @logger.info("Rendering #{file_basename}...")
-      implicit = []
-      YAML.parse_stream(rendered_content, "<rendered> #{filename}") { |d| implicit << d.implicit }
-      if rendered_content.present?
-        stream.puts "---\n" if implicit.first
-        stream.puts rendered_content
-        @logger.info("Rendered #{file_basename}")
-      else
-        @logger.warn("Rendered #{file_basename} successfully, but the result was blank")
-      end
-    rescue Psych::SyntaxError => exception
-      raise InvalidTemplateError.new("Template is not valid YAML. #{exception.message}", filename: filename)
-    end
-
-    def validate_configuration(template_sets, filenames)
-      @logger.info("Validating configuration")
-      errors = []
-      if @template_dir.present? && @template_paths.present?
-        errors << "template_dir and template_paths can not be combined"
-      elsif @template_dir.blank? && @template_paths.blank?
-        errors << "template_dir or template_paths must be set"
-      end
-
-      if filenames.present?
-        if @template_dir.nil?
-          errors << "template_dir must be set to use filenames"
-        else
-          absolute_template_dir = File.expand_path(@template_dir)
-          filenames.each do |filename|
-            absolute_file = File.expand_path(File.join(@template_dir, filename))
-            unless absolute_file.start_with?(absolute_template_dir)
-              errors << "Filename \"#{absolute_file}\" is outside the template directory," \
-              " which was resolved as #{absolute_template_dir}"
-            end
-          end
-        end
-      end
-
-      errors += template_sets.validate
-
-      unless errors.empty?
-        @logger.summary.add_action("Configuration invalid")
-        @logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
-        raise KubernetesDeploy::TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
-      end
-    end
-
-    def log_invalid_template(exception)
-      @logger.error("Failed to render #{exception.filename}")
-
-      debug_msg = ColorizedString.new("Invalid template: #{exception.filename}\n").red
-      debug_msg += "> Error message:\n#{FormattedLogger.indent_four(exception.to_s)}"
-      if exception.content
-        debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(exception.content)}"
-      end
-      @logger.summary.add_paragraph(debug_msg)
-    end
   end
 end

data/lib/kubernetes-deploy/rescue_krane_exceptions.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+require 'kubernetes-deploy/errors'
+
+module KubernetesDeploy
+  module RescueKraneExceptions
+    def run!(*args)
+      super(*args)
+    rescue Krane::DeploymentTimeoutError => e
+      raise KubernetesDeploy::DeploymentTimeoutError, e.message
+    rescue Krane::FatalDeploymentError => e
+      raise KubernetesDeploy::FatalDeploymentError, e.message
+    rescue Krane::FatalKubeAPIError => e
+      raise KubernetesDeploy::FatalKubeAPIError, e.message
+    rescue Krane::KubectlError => e
+      raise KubernetesDeploy::KubectlError, e.message
+    end
+  end
+end

data/lib/kubernetes-deploy/restart_task.rb

@@ -1,207 +1,15 @@
 # frozen_string_literal: true
-require 'kubernetes-deploy/common'
-require 'kubernetes-deploy/kubernetes_resource'
-require 'kubernetes-deploy/kubernetes_resource/deployment'
-require 'kubernetes-deploy/kubeclient_builder'
-require 'kubernetes-deploy/resource_watcher'
-require 'kubernetes-deploy/kubectl'
+require 'krane/restart_task'
+require 'kubernetes-deploy/rescue_krane_exceptions'
 
 module KubernetesDeploy
-  class RestartTask
-    class FatalRestartError < FatalDeploymentError; end
-
-    class RestartAPIError < FatalRestartError
-      def initialize(deployment_name, response)
-        super("Failed to restart #{deployment_name}. " \
-            "API returned non-200 response code (#{response.code})\n" \
-            "Response:\n#{response.body}")
-      end
-    end
-
-    HTTP_OK_RANGE = 200..299
-    ANNOTATION = "shipit.shopify.io/restart"
-
-    def initialize(context:, namespace:, logger: nil, max_watch_seconds: nil)
-      @logger = logger || KubernetesDeploy::FormattedLogger.build(namespace, context)
-      @task_config = KubernetesDeploy::TaskConfig.new(context, namespace, @logger)
-      @context = context
-      @namespace = namespace
-      @max_watch_seconds = max_watch_seconds
-    end
+  class RestartTask < ::Krane::RestartTask
+    include RescueKraneExceptions
 
     def run(*args)
-      perform!(*args)
-      true
-    rescue FatalDeploymentError
+      super(*args)
+    rescue KubernetesDeploy::FatalDeploymentError
       false
     end
-    alias_method :perform, :run
-
-    def run!(deployments_names = nil, selector: nil, verify_result: true)
-      start = Time.now.utc
-      @logger.reset
-
-      @logger.phase_heading("Initializing restart")
-      verify_config!
-      deployments = identify_target_deployments(deployments_names, selector: selector)
-
-      @logger.phase_heading("Triggering restart by touching ENV[RESTARTED_AT]")
-      patch_kubeclient_deployments(deployments)
-
-      if verify_result
-        @logger.phase_heading("Waiting for rollout")
-        resources = build_watchables(deployments, start)
-        verify_restart(resources)
-      else
-        warning = "Result verification is disabled for this task"
-        @logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
-      end
-      StatsD.distribution('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
-      @logger.print_summary(:success)
-    rescue DeploymentTimeoutError
-      StatsD.distribution('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
-      @logger.print_summary(:timed_out)
-      raise
-    rescue FatalDeploymentError => error
-      StatsD.distribution('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
-      @logger.summary.add_action(error.message) if error.message != error.class.to_s
-      @logger.print_summary(:failure)
-      raise
-    end
-    alias_method :perform!, :run!
-
-    private
-
-    def tags(status, deployments)
-      %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
-    end
-
-    def identify_target_deployments(deployment_names, selector: nil)
-      if deployment_names.nil?
-        deployments = if selector.nil?
-          @logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")
-          v1beta1_kubeclient.get_deployments(namespace: @namespace)
-        else
-          selector_string = selector.to_s
-          @logger.info(
-            "Configured to restart all deployments with the `#{ANNOTATION}` annotation and #{selector_string} selector"
-          )
-          v1beta1_kubeclient.get_deployments(namespace: @namespace, label_selector: selector_string)
-        end
-        deployments.select! { |d| d.metadata.annotations[ANNOTATION] }
-
-        if deployments.none?
-          raise FatalRestartError, "no deployments with the `#{ANNOTATION}` annotation found in namespace #{@namespace}"
-        end
-      elsif deployment_names.empty?
-        raise FatalRestartError, "Configured to restart deployments by name, but list of names was blank"
-      elsif !selector.nil?
-        raise FatalRestartError, "Can't specify deployment names and selector at the same time"
-      else
-        deployment_names = deployment_names.uniq
-        list = deployment_names.join(', ')
-        @logger.info("Configured to restart deployments by name: #{list}")
-
-        deployments = fetch_deployments(deployment_names)
-        if deployments.none?
-          raise FatalRestartError, "no deployments with names #{list} found in namespace #{@namespace}"
-        end
-      end
-      deployments
-    end
-
-    def build_watchables(kubeclient_resources, started)
-      kubeclient_resources.map do |d|
-        definition = d.to_h.deep_stringify_keys
-        r = Deployment.new(namespace: @namespace, context: @context, definition: definition, logger: @logger)
-        r.deploy_started_at = started # we don't care what happened to the resource before the restart cmd ran
-        r
-      end
-    end
-
-    def patch_deployment_with_restart(record)
-      v1beta1_kubeclient.patch_deployment(
-        record.metadata.name,
-        build_patch_payload(record),
-        @namespace
-      )
-    end
-
-    def patch_kubeclient_deployments(deployments)
-      deployments.each do |record|
-        begin
-          patch_deployment_with_restart(record)
-          @logger.info("Triggered `#{record.metadata.name}` restart")
-        rescue Kubeclient::HttpError => e
-          raise RestartAPIError.new(record.metadata.name, e.message)
-        end
-      end
-    end
-
-    def fetch_deployments(list)
-      list.map do |name|
-        record = nil
-        begin
-          record = v1beta1_kubeclient.get_deployment(name, @namespace)
-        rescue Kubeclient::ResourceNotFoundError
-          raise FatalRestartError, "Deployment `#{name}` not found in namespace `#{@namespace}`"
-        end
-        record
-      end
-    end
-
-    def build_patch_payload(deployment)
-      containers = deployment.spec.template.spec.containers
-      {
-        spec: {
-          template: {
-            spec: {
-              containers: containers.map do |container|
-                {
-                  name: container.name,
-                  env: [{ name: "RESTARTED_AT", value: Time.now.to_i.to_s }],
-                }
-              end,
-            },
-          },
-        },
-      }
-    end
-
-    def verify_restart(resources)
-      ResourceWatcher.new(resources: resources, logger: @logger, operation_name: "restart",
-        timeout: @max_watch_seconds, namespace: @namespace, context: @context).run
-      failed_resources = resources.reject(&:deploy_succeeded?)
-      success = failed_resources.empty?
-      if !success && failed_resources.all?(&:deploy_timed_out?)
-        raise DeploymentTimeoutError
-      end
-      raise FatalDeploymentError unless success
-    end
-
-    def verify_config!
-      task_config_validator = TaskConfigValidator.new(@task_config, kubectl, kubeclient_builder)
-      unless task_config_validator.valid?
-        @logger.summary.add_action("Configuration invalid")
-        @logger.summary.add_paragraph(task_config_validator.errors.map { |err| "- #{err}" }.join("\n"))
-        raise KubernetesDeploy::TaskConfigurationError
-      end
-    end
-
-    def kubeclient
-      @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
-    end
-
-    def kubectl
-      @kubectl ||= Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: true)
-    end
-
-    def v1beta1_kubeclient
-      @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
-    end
-
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
   end
 end

data/lib/kubernetes-deploy/runner_task.rb

@@ -1,193 +1,15 @@
 # frozen_string_literal: true
-require 'tempfile'
-
-require 'kubernetes-deploy/common'
-require 'kubernetes-deploy/kubeclient_builder'
-require 'kubernetes-deploy/kubectl'
-require 'kubernetes-deploy/resource_cache'
-require 'kubernetes-deploy/resource_watcher'
-require 'kubernetes-deploy/kubernetes_resource'
-require 'kubernetes-deploy/kubernetes_resource/pod'
-require 'kubernetes-deploy/runner_task_config_validator'
+require 'krane/runner_task'
+require 'kubernetes-deploy/rescue_krane_exceptions'
 
 module KubernetesDeploy
-  class RunnerTask
-    class TaskTemplateMissingError < TaskConfigurationError; end
-
-    attr_reader :pod_name
-
-    def initialize(namespace:, context:, logger: nil, max_watch_seconds: nil)
-      @logger = logger || KubernetesDeploy::FormattedLogger.build(namespace, context)
-      @task_config = KubernetesDeploy::TaskConfig.new(context, namespace, @logger)
-      @namespace = namespace
-      @context = context
-      @max_watch_seconds = max_watch_seconds
-    end
+  class RunnerTask < ::Krane::RunnerTask
+    include RescueKraneExceptions
 
     def run(*args)
-      run!(*args)
-      true
-    rescue DeploymentTimeoutError, FatalDeploymentError
+      super(*args)
+    rescue KubernetesDeploy::DeploymentTimeoutError, KubernetesDeploy::FatalDeploymentError
       false
     end
-
-    def run!(task_template:, entrypoint:, args:, env_vars: [], verify_result: true)
-      start = Time.now.utc
-      @logger.reset
-
-      @logger.phase_heading("Initializing task")
-
-      @logger.info("Validating configuration")
-      verify_config!(task_template, args)
-      @logger.info("Using namespace '#{@namespace}' in context '#{@context}'")
-
-      pod = build_pod(task_template, entrypoint, args, env_vars, verify_result)
-      validate_pod(pod)
-
-      @logger.phase_heading("Running pod")
-      create_pod(pod)
-
-      if verify_result
-        @logger.phase_heading("Streaming logs")
-        watch_pod(pod)
-      else
-        record_status_once(pod)
-      end
-      StatsD.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('success'))
-      @logger.print_summary(:success)
-    rescue DeploymentTimeoutError
-      StatsD.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('timeout'))
-      @logger.print_summary(:timed_out)
-      raise
-    rescue FatalDeploymentError
-      StatsD.distribution('task_runner.duration', StatsD.duration(start), tags: statsd_tags('failure'))
-      @logger.print_summary(:failure)
-      raise
-    end
-
-    private
-
-    def create_pod(pod)
-      @logger.info("Creating pod '#{pod.name}'")
-      pod.deploy_started_at = Time.now.utc
-      kubeclient.create_pod(pod.to_kubeclient_resource)
-      @pod_name = pod.name
-      @logger.info("Pod creation succeeded")
-    rescue Kubeclient::HttpError => e
-      msg = "Failed to create pod: #{e.class.name}: #{e.message}"
-      @logger.summary.add_paragraph(msg)
-      raise FatalDeploymentError, msg
-    end
-
-    def build_pod(template_name, entrypoint, args, env_vars, verify_result)
-      task_template = get_template(template_name)
-      @logger.info("Using template '#{template_name}'")
-      pod_template = build_pod_definition(task_template)
-      set_container_overrides!(pod_template, entrypoint, args, env_vars)
-      ensure_valid_restart_policy!(pod_template, verify_result)
-      Pod.new(namespace: @namespace, context: @context, logger: @logger, stream_logs: true,
-                    definition: pod_template.to_hash.deep_stringify_keys, statsd_tags: [])
-    end
-
-    def validate_pod(pod)
-      pod.validate_definition(kubectl)
-    end
-
-    def watch_pod(pod)
-      rw = ResourceWatcher.new(resources: [pod], logger: @logger, timeout: @max_watch_seconds,
-        operation_name: "run", namespace: @namespace, context: @context)
-      rw.run(delay_sync: 1, reminder_interval: 30.seconds)
-      raise DeploymentTimeoutError if pod.deploy_timed_out?
-      raise FatalDeploymentError if pod.deploy_failed?
-    end
-
-    def record_status_once(pod)
-      cache = ResourceCache.new(@namespace, @context, @logger)
-      pod.sync(cache)
-      warning = <<~STRING
-        #{ColorizedString.new('Result verification is disabled for this task.').yellow}
-        The following status was observed immediately after pod creation:
-        #{pod.pretty_status}
-      STRING
-      @logger.summary.add_paragraph(warning)
-    end
-
-    def verify_config!(task_template, args)
-      task_config_validator = RunnerTaskConfigValidator.new(task_template, args, @task_config, kubectl,
-        kubeclient_builder)
-      unless task_config_validator.valid?
-        @logger.summary.add_action("Configuration invalid")
-        @logger.summary.add_paragraph([task_config_validator.errors].map { |err| "- #{err}" }.join("\n"))
-        raise KubernetesDeploy::TaskConfigurationError
-      end
-    end
-
-    def get_template(template_name)
-      pod_template = kubeclient.get_pod_template(template_name, @namespace)
-      pod_template.template
-    rescue Kubeclient::ResourceNotFoundError
-      msg = "Pod template `#{template_name}` not found in namespace `#{@namespace}`, context `#{@context}`"
-      @logger.summary.add_paragraph(msg)
-      raise TaskTemplateMissingError, msg
-    rescue Kubeclient::HttpError => error
-      raise FatalKubeAPIError, "Error retrieving pod template: #{error.class.name}: #{error.message}"
-    end
-
-    def build_pod_definition(base_template)
-      pod_definition = base_template.dup
-      pod_definition.kind = 'Pod'
-      pod_definition.apiVersion = 'v1'
-      pod_definition.metadata.namespace = @namespace
-
-      unique_name = pod_definition.metadata.name + "-" + SecureRandom.hex(8)
-      @logger.warn("Name is too long, using '#{unique_name[0..62]}'") if unique_name.length > 63
-      pod_definition.metadata.name = unique_name[0..62]
-
-      pod_definition
-    end
-
-    def set_container_overrides!(pod_definition, entrypoint, args, env_vars)
-      container = pod_definition.spec.containers.find { |cont| cont.name == 'task-runner' }
-      if container.nil?
-        message = "Pod spec does not contain a template container called 'task-runner'"
-        @logger.summary.add_paragraph(message)
-        raise TaskConfigurationError, message
-      end
-
-      container.command = entrypoint if entrypoint
-      container.args = args if args
-
-      env_args = env_vars.map do |env|
-        key, value = env.split('=', 2)
-        { name: key, value: value }
-      end
-      container.env ||= []
-      container.env = container.env.map(&:to_h) + env_args
-    end
-
-    def ensure_valid_restart_policy!(template, verify)
-      restart_policy = template.spec.restartPolicy
-      if verify && restart_policy != "Never"
-        @logger.warn("Changed Pod RestartPolicy from '#{restart_policy}' to 'Never'. Disable "\
-          "result verification to use '#{restart_policy}'.")
-        template.spec.restartPolicy = "Never"
-      end
-    end
-
-    def kubectl
-      @kubectl ||= Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: true)
-    end
-
-    def kubeclient
-      @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
-    end
-
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
-
-    def statsd_tags(status)
-      %W(namespace:#{@namespace} context:#{@context} status:#{status})
-    end
   end
 end