krane 1.1.1 → 2.1.0

data/lib/krane/kubeclient_builder.rb

@@ -10,8 +10,10 @@ module Krane
       end
     end
 
-    def initialize(kubeconfig: ENV["KUBECONFIG"])
-      files = kubeconfig || "#{Dir.home}/.kube/config"
+    attr_reader :kubeconfig_files
+
+    def initialize(kubeconfig: nil)
+      files = kubeconfig || ENV["KUBECONFIG"] || "#{Dir.home}/.kube/config"
       # Split the list by colon for Linux and Mac, and semicolon for Windows.
       @kubeconfig_files = files.split(/[:;]/).map!(&:strip).reject(&:empty?)
     end

data/lib/krane/kubectl.rb

@@ -6,6 +6,8 @@ module Krane
     ERROR_MATCHERS = {
       not_found: /NotFound/,
       client_timeout: /Client\.Timeout exceeded while awaiting headers/,
+      empty: /\A\z/,
+      context_deadline: /context deadline exceeded/,
     }
     DEFAULT_TIMEOUT = 15
     MAX_RETRY_DELAY = 16
@@ -13,7 +15,7 @@ module Krane
 
     class ResourceNotFoundError < StandardError; end
 
-    delegate :namespace, :context, :logger, to: :@task_config
+    delegate :namespace, :context, :logger, :kubeconfig, to: :@task_config
 
     def initialize(task_config:, log_failure_by_default:, default_timeout: DEFAULT_TIMEOUT,
       output_is_sensitive_default: false)
@@ -33,8 +35,18 @@ module Krane
 
       (1..attempts).to_a.each do |current_attempt|
         logger.debug("Running command (attempt #{current_attempt}): #{cmd.join(' ')}")
-        out, err, st = Open3.capture3(*cmd)
-        logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) unless output_is_sensitive
+        env = { 'KUBECONFIG' => kubeconfig }
+        out, err, st = Open3.capture3(env, *cmd)
+
+        # https://github.com/Shopify/krane/issues/395
+        unless out.valid_encoding?
+          out = out.dup.force_encoding(Encoding::UTF_8)
+        end
+
+        if logger.debug? && !output_is_sensitive
+          # don't do the gsub unless we're going to print this
+          logger.debug("Kubectl out: " + out.gsub(/\s+/, ' '))
+        end
 
         break if st.success?
         raise(ResourceNotFoundError, err) if err.match(ERROR_MATCHERS[:not_found]) && raise_if_not_found
@@ -52,7 +64,8 @@ module Krane
         else
           logger.debug("Kubectl err: #{output_is_sensitive ? '<suppressed sensitive output>' : err}")
         end
-        StatsD.client.increment('kubectl.error', 1, tags: { context: context, namespace: namespace, cmd: cmd[1] })
+        StatsD.client.increment('kubectl.error', 1, tags: { context: context, namespace: namespace, cmd: cmd[1],
+                                                            max_attempt: attempts, current_attempt: current_attempt })
 
         break unless retriable_err?(err, retry_whitelist) && current_attempt < attempts
         sleep(retry_delay(current_attempt))
@@ -69,7 +82,7 @@ module Krane
     def version_info
       @version_info ||=
         begin
-          response, _, status = run("version", use_namespace: false, log_failure: true)
+          response, _, status = run("version", use_namespace: false, log_failure: true, attempts: 2)
           raise KubectlError, "Could not retrieve kubectl version info" unless status.success?
           extract_version_info_from_kubectl_response(response)
         end

data/lib/krane/kubernetes_resource.rb

@@ -6,9 +6,12 @@ require 'krane/remote_logs'
 require 'krane/duration_parser'
 require 'krane/label_selector'
 require 'krane/rollout_conditions'
+require 'krane/psych_k8s_compatibility'
 
 module Krane
   class KubernetesResource
+    using PsychK8sCompatibility
+
     attr_reader :name, :namespace, :context
     attr_writer :type, :deploy_started_at, :global
 
@@ -32,9 +35,9 @@ module Krane
       If you have reason to believe it will succeed, retry the deploy to continue to monitor the rollout.
       MSG
 
-    TIMEOUT_OVERRIDE_ANNOTATION_SUFFIX = "timeout-override"
-    TIMEOUT_OVERRIDE_ANNOTATION_DEPRECATED = "kubernetes-deploy.shopify.io/#{TIMEOUT_OVERRIDE_ANNOTATION_SUFFIX}"
-    TIMEOUT_OVERRIDE_ANNOTATION = "krane.shopify.io/#{TIMEOUT_OVERRIDE_ANNOTATION_SUFFIX}"
+    ALLOWED_DEPLOY_METHOD_OVERRIDES = %w(create replace replace-force)
+    DEPLOY_METHOD_OVERRIDE_ANNOTATION = "deploy-method-override"
+    TIMEOUT_OVERRIDE_ANNOTATION = "timeout-override"
     LAST_APPLIED_ANNOTATION = "kubectl.kubernetes.io/last-applied-configuration"
     SENSITIVE_TEMPLATE_CONTENT = false
     SERVER_DRY_RUNNABLE = false
@@ -60,8 +63,8 @@ module Krane
       end
 
       def class_for_kind(kind)
-        if Krane.const_defined?(kind)
-          Krane.const_get(kind)
+        if Krane.const_defined?(kind, false)
+          Krane.const_get(kind, false)
         end
       rescue NameError
         nil
@@ -103,7 +106,7 @@ module Krane
     def timeout_override
       return @timeout_override if defined?(@timeout_override)
 
-      @timeout_override = DurationParser.new(krane_annotation_value(TIMEOUT_OVERRIDE_ANNOTATION_SUFFIX)).parse!.to_i
+      @timeout_override = DurationParser.new(krane_annotation_value(TIMEOUT_OVERRIDE_ANNOTATION)).parse!.to_i
     rescue DurationParser::ParsingError
       @timeout_override = nil
     end
@@ -123,7 +126,6 @@ module Krane
       @statsd_report_done = false
       @disappeared = false
       @validation_errors = []
-      @validation_warnings = []
       @instance_data = {}
       @server_dry_run_validated = false
     end
@@ -134,22 +136,13 @@ module Krane
 
     def validate_definition(kubectl, selector: nil)
       @validation_errors = []
-      @validation_warnings = []
       validate_selector(selector) if selector
       validate_timeout_annotation
-      validate_annotation_version
+      validate_deploy_method_override_annotation
       validate_spec_with_kubectl(kubectl)
       @validation_errors.present?
     end
 
-    def validation_warning_msg
-      @validation_warnings.join("\n")
-    end
-
-    def has_warnings?
-      @validation_warnings.present?
-    end
-
     def validation_error_msg
       @validation_errors.join("\n")
     end
@@ -228,6 +221,11 @@ module Krane
       @type || self.class.kind
     end
 
+    def group
+      grouping, version = @definition.dig("apiVersion").split("/")
+      version ? grouping : "core"
+    end
+
     def kubectl_resource_type
       type
     end
@@ -242,10 +240,14 @@ module Krane
       if @definition.dig("metadata", "name").blank? && uses_generate_name?
         :create
       else
-        :apply
+        deploy_method_override || :apply
       end
     end
 
+    def deploy_method_override
+      krane_annotation_value(DEPLOY_METHOD_OVERRIDE_ANNOTATION)&.to_sym
+    end
+
     def sync_debug_info(kubectl)
       @debug_events = fetch_events(kubectl) unless ENV[DISABLE_FETCHING_EVENT_INFO]
       @debug_logs = fetch_debug_logs if print_debug_logs? && !ENV[DISABLE_FETCHING_LOG_INFO]
@@ -495,8 +497,8 @@ module Krane
     private
 
     def validate_timeout_annotation
-      timeout_override_value = krane_annotation_value(TIMEOUT_OVERRIDE_ANNOTATION_SUFFIX)
-      timeout_annotation_key = krane_annotation_key(TIMEOUT_OVERRIDE_ANNOTATION_SUFFIX)
+      timeout_override_value = krane_annotation_value(TIMEOUT_OVERRIDE_ANNOTATION)
+      timeout_annotation_key = Annotation.for(TIMEOUT_OVERRIDE_ANNOTATION)
       return if timeout_override_value.nil?
 
       override = DurationParser.new(timeout_override_value).parse!
@@ -509,30 +511,18 @@ module Krane
       @validation_errors << "#{timeout_annotation_key} annotation is invalid: #{e}"
     end
 
-    def validate_annotation_version
-      return if validation_warning_msg.include?("annotations is deprecated")
-      annotation_keys = @definition.dig("metadata", "annotations")&.keys
-      annotation_keys&.each do |annotation|
-        if annotation.include?("kubernetes-deploy.shopify.io")
-          annotation_prefix = annotation.split('/').first
-          @validation_warnings << "#{annotation_prefix} as a prefix for annotations is deprecated: "\
-            "Use the 'krane.shopify.io' annotation prefix instead"
-          return
-        end
+    def validate_deploy_method_override_annotation
+      deploy_method_override_value = krane_annotation_value(DEPLOY_METHOD_OVERRIDE_ANNOTATION)
+      deploy_method_override_annotation_key = Annotation.for(DEPLOY_METHOD_OVERRIDE_ANNOTATION)
+      return unless deploy_method_override_value
+      unless ALLOWED_DEPLOY_METHOD_OVERRIDES.include?(deploy_method_override_value)
+        @validation_errors << "#{deploy_method_override_annotation_key} is invalid: Accepted values are: " \
+          "#{ALLOWED_DEPLOY_METHOD_OVERRIDES.join(', ')} but got #{deploy_method_override_value}"
       end
     end
 
     def krane_annotation_value(suffix)
-      @definition.dig("metadata", "annotations", "kubernetes-deploy.shopify.io/#{suffix}") ||
-        @definition.dig("metadata", "annotations", "krane.shopify.io/#{suffix}")
-    end
-
-    def krane_annotation_key(suffix)
-      if @definition.dig("metadata", "annotations", "kubernetes-deploy.shopify.io/#{suffix}")
-        "kubernetes-deploy.shopify.io/#{suffix}"
-      elsif @definition.dig("metadata", "annotations", "krane.shopify.io/#{suffix}")
-        "krane.shopify.io/#{suffix}"
-      end
+      @definition.dig("metadata", "annotations", Annotation.for(suffix))
     end
 
     def validate_selector(selector)
@@ -572,7 +562,7 @@ module Krane
     def validate_with_server_side_dry_run(kubectl)
       command = ["apply", "-f", file_path, "--server-dry-run", "--output=name"]
       kubectl.run(*command, log_failure: false, output_is_sensitive: sensitive_template_content?,
-        retry_whitelist: [:client_timeout], attempts: 3)
+        retry_whitelist: [:client_timeout, :empty, :context_deadline], attempts: 3)
     end
 
     # Local dry run is supported on only create and apply
@@ -582,7 +572,7 @@ module Krane
       verb = deploy_method == :apply ? "apply" : "create"
       command = [verb, "-f", file_path, "--dry-run", "--output=name"]
       kubectl.run(*command, log_failure: false, output_is_sensitive: sensitive_template_content?,
-        retry_whitelist: [:client_timeout], attempts: 3, use_namespace: !global?)
+        retry_whitelist: [:client_timeout, :empty, :context_deadline], attempts: 3, use_namespace: !global?)
     end
 
     def labels

data/lib/krane/kubernetes_resource/custom_resource.rb

@@ -62,7 +62,7 @@ module Krane
       kind
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       @crd.validate_rollout_conditions
@@ -70,7 +70,7 @@ module Krane
       @validation_errors << "The CRD that specifies this resource is using invalid rollout conditions. " \
       "Krane will not be able to continue until those rollout conditions are fixed.\n" \
       "Rollout conditions can be found on the CRD that defines this resource (#{@crd.name}), " \
-      "under the annotation #{CustomResourceDefinition::ROLLOUT_CONDITIONS_ANNOTATION}.\n" \
+      "under the annotation #{Annotation.for(CustomResourceDefinition::ROLLOUT_CONDITIONS_ANNOTATION)}.\n" \
       "Validation failed with: #{e}"
     end
 

data/lib/krane/kubernetes_resource/custom_resource_definition.rb

@@ -2,9 +2,8 @@
 module Krane
   class CustomResourceDefinition < KubernetesResource
     TIMEOUT = 2.minutes
-    ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX = "instance-rollout-conditions"
-    ROLLOUT_CONDITIONS_ANNOTATION = "krane.shopify.io/#{ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX}"
-    TIMEOUT_FOR_INSTANCE_ANNOTATION = "krane.shopify.io/instance-timeout"
+    ROLLOUT_CONDITIONS_ANNOTATION = "instance-rollout-conditions"
+    TIMEOUT_FOR_INSTANCE_ANNOTATION = "instance-timeout"
     GLOBAL = true
 
     def deploy_succeeded?
@@ -20,7 +19,7 @@ module Krane
     end
 
     def timeout_for_instance
-      timeout = krane_annotation_value("instance-timeout")
+      timeout = krane_annotation_value(TIMEOUT_FOR_INSTANCE_ANNOTATION)
       DurationParser.new(timeout).parse!.to_i
     rescue DurationParser::ParsingError
       nil
@@ -46,6 +45,10 @@ module Krane
       @definition.dig("spec", "names", "kind")
     end
 
+    def group
+      @definition.dig("spec", "group")
+    end
+
     def prunable?
       prunable = krane_annotation_value("prunable")
       prunable == "true"
@@ -59,25 +62,25 @@ module Krane
     def rollout_conditions
       return @rollout_conditions if defined?(@rollout_conditions)
 
-      @rollout_conditions = if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX)
-        RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX))
+      @rollout_conditions = if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION)
+        RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION))
       end
     rescue RolloutConditionsError
       @rollout_conditions = nil
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       validate_rollout_conditions
     rescue RolloutConditionsError => e
-      @validation_errors << "Annotation #{krane_annotation_key(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX)} "\
+      @validation_errors << "Annotation #{Annotation.for(ROLLOUT_CONDITIONS_ANNOTATION)} " \
         "on #{name} is invalid: #{e}"
     end
 
     def validate_rollout_conditions
-      if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX) && @rollout_conditions_validated.nil?
-        conditions = RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION_SUFFIX))
+      if krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION) && @rollout_conditions_validated.nil?
+        conditions = RolloutConditions.from_annotation(krane_annotation_value(ROLLOUT_CONDITIONS_ANNOTATION))
         conditions.validate!
       end
 

data/lib/krane/kubernetes_resource/deployment.rb

@@ -5,9 +5,7 @@ module Krane
   class Deployment < KubernetesResource
     TIMEOUT = 7.minutes
     SYNC_DEPENDENCIES = %w(Pod ReplicaSet)
-    REQUIRED_ROLLOUT_ANNOTATION_SUFFIX = "required-rollout"
-    REQUIRED_ROLLOUT_ANNOTATION_DEPRECATED = "kubernetes-deploy.shopify.io/#{REQUIRED_ROLLOUT_ANNOTATION_SUFFIX}"
-    REQUIRED_ROLLOUT_ANNOTATION = "krane.shopify.io/#{REQUIRED_ROLLOUT_ANNOTATION_SUFFIX}"
+    REQUIRED_ROLLOUT_ANNOTATION = "required-rollout"
     REQUIRED_ROLLOUT_TYPES = %w(maxUnavailable full none).freeze
     DEFAULT_REQUIRED_ROLLOUT = 'full'
 
@@ -97,7 +95,7 @@ module Krane
       progress_condition.present? ? deploy_failing_to_progress? : super
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       unless REQUIRED_ROLLOUT_TYPES.include?(required_rollout) || percent?(required_rollout)
@@ -106,7 +104,7 @@ module Krane
 
       strategy = @definition.dig('spec', 'strategy', 'type').to_s
       if required_rollout.downcase == 'maxunavailable' && strategy.present? && strategy.downcase != 'rollingupdate'
-        @validation_errors << "'#{krane_annotation_key(REQUIRED_ROLLOUT_ANNOTATION_SUFFIX)}: #{required_rollout}' "\
+        @validation_errors << "'#{Annotation.for(REQUIRED_ROLLOUT_ANNOTATION)}: #{required_rollout}' " \
           "is incompatible with strategy '#{strategy}'"
       end
 
@@ -151,7 +149,7 @@ module Krane
     end
 
     def rollout_annotation_err_msg
-      "'#{krane_annotation_key(REQUIRED_ROLLOUT_ANNOTATION_SUFFIX)}: #{required_rollout}' is invalid. "\
+      "'#{Annotation.for(REQUIRED_ROLLOUT_ANNOTATION)}: #{required_rollout}' is invalid. " \
         "Acceptable values: #{REQUIRED_ROLLOUT_TYPES.join(', ')}"
     end
 
@@ -191,7 +189,7 @@ module Krane
     def min_available_replicas
       if percent?(required_rollout)
         (desired_replicas * required_rollout.to_i / 100.0).ceil
-      elsif max_unavailable =~ /%/
+      elsif max_unavailable.is_a?(String) && max_unavailable =~ /%/
         (desired_replicas * (100 - max_unavailable.to_i) / 100.0).ceil
       else
         desired_replicas - max_unavailable.to_i

data/lib/krane/kubernetes_resource/persistent_volume_claim.rb

@@ -63,6 +63,7 @@ module Krane
       attr_reader :name
 
       def initialize(definition)
+        super(definition: definition, namespace: nil, context: nil, logger: nil)
         @definition = definition
         @name = definition.dig("metadata", "name").to_s
       end

data/lib/krane/kubernetes_resource/pod.rb

@@ -219,14 +219,7 @@ module Krane
         limbo_reason = @status.dig("state", "waiting", "reason")
         limbo_message = @status.dig("state", "waiting", "message")
 
-        if @status.dig("lastState", "terminated", "reason") == "ContainerCannotRun"
-          # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
-          exit_code = @status.dig('lastState', 'terminated', 'exitCode')
-          "Failed to start (exit #{exit_code}): #{@status.dig('lastState', 'terminated', 'message')}"
-        elsif @status.dig("state", "terminated", "reason") == "ContainerCannotRun"
-          exit_code = @status.dig('state', 'terminated', 'exitCode')
-          "Failed to start (exit #{exit_code}): #{@status.dig('state', 'terminated', 'message')}"
-        elsif limbo_reason == "CrashLoopBackOff"
+        if limbo_reason == "CrashLoopBackOff"
           exit_code = @status.dig('lastState', 'terminated', 'exitCode')
           "Crashing repeatedly (exit #{exit_code}). See logs for more information."
         elsif limbo_reason == "ErrImagePull" && limbo_message.match(/not found/i)
@@ -234,6 +227,17 @@ module Krane
           "Did you wait for it to be built and pushed to the registry before deploying?"
         elsif limbo_reason == "CreateContainerConfigError"
           "Failed to generate container configuration: #{limbo_message}"
+        elsif @status.dig("lastState", "terminated", "reason") == "ContainerCannotRun"
+          # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
+          exit_code = @status.dig('lastState', 'terminated', 'exitCode')
+          # We've observed failures here that are actually issues with the node or kube infra, and not with the
+          # container. These issues have been transient and result in a 128 exit code, so do not treat these as fatal.
+          return if exit_code == 128
+          "Failed to start (exit #{exit_code}): #{@status.dig('lastState', 'terminated', 'message')}"
+        elsif @status.dig("state", "terminated", "reason") == "ContainerCannotRun"
+          exit_code = @status.dig('state', 'terminated', 'exitCode')
+          return if exit_code == 128
+          "Failed to start (exit #{exit_code}): #{@status.dig('state', 'terminated', 'message')}"
         end
       end
 

data/lib/krane/psych_k8s_compatibility.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'psych'
+
+module PsychK8sCompatibility
+  def self.massage_node(n)
+    if n.is_a?(Psych::Nodes::Scalar) &&
+        (n.style == Psych::Nodes::Scalar::PLAIN) &&
+        n.value.is_a?(String) &&
+        n.value =~ /\A[+-]?\d+(?:\.\d+)?[eE][+-]?\d+\z/
+      n.style = Psych::Nodes::Scalar::DOUBLE_QUOTED
+    end
+  end
+
+  refine Psych.singleton_class do
+    def dump(o, io = nil, options = {})
+      if io.is_a?(Hash)
+        options = io
+        io = nil
+      end
+      visitor = Psych::Visitors::YAMLTree.create(options)
+      visitor << o
+      visitor.tree.each { |n| PsychK8sCompatibility.massage_node(n) }
+      visitor.tree.yaml(io, options)
+    end
+
+    def dump_stream(*objects)
+      visitor = Psych::Visitors::YAMLTree.create({})
+      objects.each do |o|
+        visitor << o
+      end
+      visitor.tree.each { |n| PsychK8sCompatibility.massage_node(n) }
+      visitor.tree.yaml
+    end
+  end
+end

data/lib/krane/render_task.rb

@@ -24,8 +24,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue Krane::FatalDeploymentError
       false