kubernetes-deploy 0.25.0 → 0.26.0

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -29,6 +29,8 @@ module KubernetesDeploy
       MSG
 
     TIMEOUT_OVERRIDE_ANNOTATION = "kubernetes-deploy.shopify.io/timeout-override"
+    LAST_APPLIED_ANNOTATION = "kubectl.kubernetes.io/last-applied-configuration"
+    KUBECTL_OUTPUT_IS_SENSITIVE = false
 
     class << self
       def build(namespace:, context:, definition:, logger:, statsd_tags:, crd: nil)
@@ -37,12 +39,8 @@ module KubernetesDeploy
         if definition["kind"].blank?
           raise InvalidTemplateError.new("Template missing 'Kind'", content: definition.to_yaml)
         end
-        begin
-          if KubernetesDeploy.const_defined?(definition["kind"])
-            klass = KubernetesDeploy.const_get(definition["kind"])
-            return klass.new(**opts)
-          end
-        rescue NameError
+        if (klass = class_for_kind(definition["kind"]))
+          return klass.new(**opts)
         end
         if crd
           CustomResource.new(crd: crd, **opts)
@@ -53,6 +51,14 @@ module KubernetesDeploy
         end
       end
 
+      def class_for_kind(kind)
+        if KubernetesDeploy.const_defined?(kind)
+          KubernetesDeploy.const_get(kind) # rubocop:disable Sorbet/ConstantsFromStrings
+        end
+      rescue NameError
+        nil
+      end
+
       def timeout
         self::TIMEOUT
       end
@@ -101,14 +107,21 @@ module KubernetesDeploy
       Kubeclient::Resource.new(@definition)
     end
 
-    def validate_definition(kubectl)
+    def validate_definition(kubectl, selector: nil)
       @validation_errors = []
+      validate_selector(selector) if selector
       validate_timeout_annotation
 
       command = ["create", "-f", file_path, "--dry-run", "--output=name"]
-      _, err, st = kubectl.run(*command, log_failure: false)
+      _, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: kubectl_output_is_sensitive?)
       return true if st.success?
-      @validation_errors << err
+      if kubectl_output_is_sensitive?
+        @validation_errors << <<-EOS
+          Validation for #{id} failed. Detailed information is unavailable as the raw error may contain sensitive data.
+        EOS
+      else
+        @validation_errors << err
+      end
       false
     end
 
@@ -124,6 +137,10 @@ module KubernetesDeploy
       "#{type}/#{name}"
     end
 
+    def <=>(other)
+      id <=> other.id
+    end
+
     def file_path
       file.path
     end
@@ -305,6 +322,10 @@ module KubernetesDeploy
       end
     end
 
+    def kubectl_output_is_sensitive?
+      self.class::KUBECTL_OUTPUT_IS_SENSITIVE
+    end
+
     class Event
       EVENT_SEPARATOR = "ENDEVENT--BEGINEVENT"
       FIELD_SEPARATOR = "ENDFIELD--BEGINFIELD"
@@ -388,6 +409,23 @@ module KubernetesDeploy
       @definition.dig("metadata", "annotations", TIMEOUT_OVERRIDE_ANNOTATION)
     end
 
+    def validate_selector(selector)
+      if labels.nil?
+        @validation_errors << "selector #{selector} passed in, but no labels were defined"
+        return
+      end
+
+      unless selector.to_h <= labels
+        label_name = 'label'.pluralize(labels.size)
+        label_string = LabelSelector.new(labels).to_s
+        @validation_errors << "selector #{selector} does not match #{label_name} #{label_string}"
+      end
+    end
+
+    def labels
+      @definition.dig("metadata", "labels")
+    end
+
     def file
       @file ||= create_definition_tempfile
     end

data/lib/kubernetes-deploy/kubernetes_resource/custom_resource.rb

@@ -62,7 +62,7 @@ module KubernetesDeploy
       kind
     end
 
-    def validate_definition(kubectl)
+    def validate_definition(*)
       super
 
       @crd.validate_rollout_conditions

data/lib/kubernetes-deploy/kubernetes_resource/custom_resource_definition.rb

@@ -66,7 +66,7 @@ module KubernetesDeploy
       @rollout_conditions = nil
     end
 
-    def validate_definition(_)
+    def validate_definition(*)
       super
 
       validate_rollout_conditions

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -92,7 +92,7 @@ module KubernetesDeploy
       progress_condition.present? ? deploy_failing_to_progress? : super
     end
 
-    def validate_definition(_)
+    def validate_definition(*)
       super
 
       unless REQUIRED_ROLLOUT_TYPES.include?(required_rollout) || percent?(required_rollout)

data/lib/kubernetes-deploy/kubernetes_resource/horizontal_pod_autoscaler.rb

@@ -2,16 +2,17 @@
 module KubernetesDeploy
   class HorizontalPodAutoscaler < KubernetesResource
     TIMEOUT = 3.minutes
-    RECOVERABLE_CONDITIONS = %w(ScalingDisabled FailedGet)
+    RECOVERABLE_CONDITION_PREFIX = "FailedGet"
 
     def deploy_succeeded?
-      scaling_active_condition["status"] == "True"
+      scaling_active_condition["status"] == "True" || scaling_disabled?
     end
 
     def deploy_failed?
       return false unless exists?
-      recoverable = RECOVERABLE_CONDITIONS.any? { |c| scaling_active_condition.fetch("reason", "").start_with?(c) }
-      scaling_active_condition["status"] == "False" && !recoverable
+      return false if scaling_disabled?
+      scaling_active_condition["status"] == "False" &&
+      !scaling_active_condition.fetch("reason", "").start_with?(RECOVERABLE_CONDITION_PREFIX)
     end
 
     def kubectl_resource_type
@@ -21,6 +22,8 @@ module KubernetesDeploy
     def status
       if !exists?
         super
+      elsif scaling_disabled?
+        "ScalingDisabled"
       elsif deploy_succeeded?
         "Configured"
       elsif scaling_active_condition.present? || able_to_scale_condition.present?
@@ -42,6 +45,11 @@ module KubernetesDeploy
 
     private
 
+    def scaling_disabled?
+      scaling_active_condition["status"] == "False" &&
+      scaling_active_condition["reason"] == "ScalingDisabled"
+    end
+
     def conditions
       @instance_data.dig("status", "conditions") || []
     end

data/lib/kubernetes-deploy/kubernetes_resource/network_policy.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class NetworkPolicy < KubernetesResource
+    TIMEOUT = 30.seconds
+
+    def status
+      exists? ? "Created" : "Not Found"
+    end
+
+    def deploy_succeeded?
+      exists?
+    end
+
+    def deploy_failed?
+      false
+    end
+
+    def timeout_message
+      UNUSUAL_FAILURE_MESSAGE
+    end
+  end
+end

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -9,6 +9,8 @@ module KubernetesDeploy
       Preempting
     )
 
+    attr_accessor :stream_logs
+
     def initialize(namespace:, context:, definition:, logger:,
       statsd_tags: nil, parent: nil, deploy_started_at: nil, stream_logs: false)
       @parent = parent

data/lib/kubernetes-deploy/kubernetes_resource/secret.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class Secret < KubernetesResource
+    TIMEOUT = 30.seconds
+    KUBECTL_OUTPUT_IS_SENSITIVE = true
+
+    def status
+      exists? ? "Available" : "Not Found"
+    end
+
+    def deploy_succeeded?
+      exists?
+    end
+
+    def deploy_failed?
+      false
+    end
+
+    def timeout_message
+      UNUSUAL_FAILURE_MESSAGE
+    end
+  end
+end

data/lib/kubernetes-deploy/label_selector.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module KubernetesDeploy
+  class LabelSelector
+    def self.parse(string)
+      selector = {}
+
+      string.split(',').each do |kvp|
+        key, value = kvp.split('=', 2)
+
+        if key.blank?
+          raise ArgumentError, "key is blank"
+        end
+
+        if key.end_with?("!")
+          raise ArgumentError, "!= selectors are not supported"
+        end
+
+        if value&.start_with?("=")
+          raise ArgumentError, "== selectors are not supported"
+        end
+
+        selector[key] = value
+      end
+
+      new(selector)
+    end
+
+    def initialize(hash)
+      @selector = hash
+    end
+
+    def to_h
+      @selector
+    end
+
+    def to_s
+      return "" if @selector.nil?
+      @selector.map { |k, v| "#{k}=#{v}" }.join(",")
+    end
+  end
+end

data/lib/kubernetes-deploy/options_helper.rb

@@ -2,25 +2,41 @@
 
 module KubernetesDeploy
   module OptionsHelper
-    def self.default_and_check_template_dir(template_dir)
-      if !template_dir && ENV.key?("ENVIRONMENT")
-        template_dir = "config/deploy/#{ENV['ENVIRONMENT']}"
-      end
+    class OptionsError < StandardError; end
 
-      if !template_dir || template_dir.empty?
-        puts "Template directory is unknown. " \
-          "Either specify --template-dir argument or set $ENVIRONMENT to use config/deploy/$ENVIRONMENT " \
-        + "as a default path."
-        exit(1)
+    STDIN_TEMP_FILE = "from_stdin.yml.erb"
+    class << self
+      def with_validated_template_dir(template_dir)
+        if template_dir == '-'
+          Dir.mktmpdir("kubernetes-deploy") do |dir|
+            template_dir_from_stdin(temp_dir: dir)
+            yield dir
+          end
+        else
+          yield default_template_dir(template_dir)
+        end
       end
 
-      template_dir
-    end
+      private
+
+      def default_template_dir(template_dir)
+        if ENV.key?("ENVIRONMENT")
+          template_dir = File.join("config", "deploy", ENV['ENVIRONMENT'])
+        end
+
+        if !template_dir || template_dir.empty?
+          raise OptionsError, "Template directory is unknown. " \
+            "Either specify --template-dir argument or set $ENVIRONMENT to use config/deploy/$ENVIRONMENT " \
+            "as a default path."
+        end
+
+        template_dir
+      end
 
-    def self.revision_from_environment
-      ENV.fetch('REVISION') do
-        puts "ENV['REVISION'] is missing. Please specify the commit SHA"
-        exit 1
+      def template_dir_from_stdin(temp_dir:)
+        File.open(File.join(temp_dir, STDIN_TEMP_FILE), 'w+') { |f| f.print($stdin.read) }
+      rescue IOError, Errno::ENOENT => e
+        raise OptionsError, e.message
       end
     end
   end

data/lib/kubernetes-deploy/remote_logs.rb

@@ -28,7 +28,12 @@ module KubernetesDeploy
     end
 
     def print_latest
-      @container_logs.each { |cl| cl.print_latest(prefix: @container_logs.length > 1) }
+      @container_logs.each do |cl|
+        unless cl.printing_started?
+          @logger.info("Streaming logs from #{@parent_id} container '#{cl.container_name}':")
+        end
+        cl.print_latest(prefix: @container_logs.length > 1)
+      end
     end
 
     def print_all(prevent_duplicate: true)

data/lib/kubernetes-deploy/renderer.rb

@@ -24,7 +24,11 @@ module KubernetesDeploy
       @logger = logger
       @bindings = bindings
       # Max length of podname is only 63chars so try to save some room by truncating sha to 8 chars
-      @id = current_sha[0...8] + "-#{SecureRandom.hex(4)}" if current_sha
+      @id = if ENV["TASK_ID"]
+        ENV["TASK_ID"]
+      elsif current_sha
+        current_sha[0...8] + "-#{SecureRandom.hex(4)}"
+      end
     end
 
     def render_template(filename, raw_template)

data/lib/kubernetes-deploy/resource_cache.rb

@@ -50,7 +50,10 @@ module KubernetesDeploy
     end
 
     def fetch_by_kind(kind)
-      raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", "--output=json", attempts: 5)
+      resource_class = KubernetesResource.class_for_kind(kind)
+      output_is_sensitive = resource_class.nil? ? false : resource_class::KUBECTL_OUTPUT_IS_SENSITIVE
+      raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json",
+         output_is_sensitive: output_is_sensitive)
       raise KubectlError unless st.success?
 
       instances = {}

data/lib/kubernetes-deploy/restart_task.rb

@@ -5,8 +5,6 @@ require 'kubernetes-deploy/kubectl'
 
 module KubernetesDeploy
   class RestartTask
-    include KubernetesDeploy::KubeclientBuilder
-
     class FatalRestartError < FatalDeploymentError; end
 
     class RestartAPIError < FatalRestartError
@@ -34,13 +32,13 @@ module KubernetesDeploy
       false
     end
 
-    def perform!(deployments_names = nil)
+    def perform!(deployments_names = nil, selector: nil)
       start = Time.now.utc
       @logger.reset
 
       @logger.phase_heading("Initializing restart")
       verify_namespace
-      deployments = identify_target_deployments(deployments_names)
+      deployments = identify_target_deployments(deployments_names, selector: selector)
       if kubectl.server_version < Gem::Version.new(MIN_KUBE_VERSION)
         @logger.warn(KubernetesDeploy::Errors.server_version_warning(kubectl.server_version))
       end
@@ -76,17 +74,27 @@ module KubernetesDeploy
       %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
     end
 
-    def identify_target_deployments(deployment_names)
+    def identify_target_deployments(deployment_names, selector: nil)
       if deployment_names.nil?
-        @logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")
-        deployments = v1beta1_kubeclient.get_deployments(namespace: @namespace)
-          .select { |d| d.metadata.annotations[ANNOTATION] }
+        deployments = if selector.nil?
+          @logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")
+          v1beta1_kubeclient.get_deployments(namespace: @namespace)
+        else
+          selector_string = selector.to_s
+          @logger.info(
+            "Configured to restart all deployments with the `#{ANNOTATION}` annotation and #{selector_string} selector"
+          )
+          v1beta1_kubeclient.get_deployments(namespace: @namespace, label_selector: selector_string)
+        end
+        deployments.select! { |d| d.metadata.annotations[ANNOTATION] }
 
         if deployments.none?
           raise FatalRestartError, "no deployments with the `#{ANNOTATION}` annotation found in namespace #{@namespace}"
         end
       elsif deployment_names.empty?
         raise FatalRestartError, "Configured to restart deployments by name, but list of names was blank"
+      elsif !selector.nil?
+        raise FatalRestartError, "Can't specify deployment names and selector at the same time"
       else
         deployment_names = deployment_names.uniq
         list = deployment_names.join(', ')
@@ -166,7 +174,7 @@ module KubernetesDeploy
     end
 
     def kubeclient
-      @kubeclient ||= build_v1_kubeclient(@context)
+      @kubeclient ||= kubeclient_builder.build_v1_kubeclient(@context)
     end
 
     def kubectl
@@ -174,7 +182,11 @@ module KubernetesDeploy
     end
 
     def v1beta1_kubeclient
-      @v1beta1_kubeclient ||= build_v1beta1_kubeclient(@context)
+      @v1beta1_kubeclient ||= kubeclient_builder.build_v1beta1_kubeclient(@context)
+    end
+
+    def kubeclient_builder
+      @kubeclient_builder ||= KubeclientBuilder.new
     end
   end
 end