krane 1.0.0 → 1.1.4

data/lib/krane/cli/global_deploy_command.rb

@@ -8,7 +8,8 @@ module Krane
         "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
                          aliases: :f, required: false, default: [],
                          desc: "Directories and files that contains the configuration to apply" },
-        "stdin" => { type: :boolean, default: false, desc: "Read resources from stdin" },
+        "stdin" => { type: :boolean, default: false,
+                     desc: "[DEPRECATED] Read resources from stdin" },
         "global-timeout" => { type: :string, banner: "duration", default: DEFAULT_DEPLOY_TIMEOUT,
                               desc: "Max duration to monitor workloads correctly deployed" },
         "verify-result" => { type: :boolean, default: true,
@@ -28,11 +29,10 @@ module Krane
 
         selector = ::Krane::LabelSelector.parse(options[:selector])
 
-        # never mutate options directly
         filenames = options[:filenames].dup
         filenames << "-" if options[:stdin]
         if filenames.empty?
-          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+          raise(Thor::RequiredArgumentMissingError, '--filenames must be set and not empty')
         end
 
         ::Krane::OptionsHelper.with_processed_template_paths(filenames) do |paths|

data/lib/krane/cli/render_command.rb

@@ -7,7 +7,8 @@ module Krane
         "bindings" => { type: :array, banner: "foo=bar abc=def", desc: 'Bindings for erb' },
         "filenames" => { type: :array, banner: 'config/deploy/production config/deploy/my-extra-resource.yml',
                          required: false, default: [], aliases: 'f', desc: 'Directories and files to render' },
-        "stdin" => { type: :boolean, desc: "Read resources from stdin", default: false },
+        "stdin" => { type: :boolean, default: false,
+                     desc: "[DEPRECATED] Read resources from stdin" },
         "current-sha" => { type: :string, banner: "SHA", desc: "Expose SHA `current_sha` in ERB bindings",
                            lazy_default: '' },
       }
@@ -20,11 +21,10 @@ module Krane
         bindings_parser = ::Krane::BindingsParser.new
         options[:bindings]&.each { |b| bindings_parser.add(b) }
 
-        # never mutate options directly
         filenames = options[:filenames].dup
         filenames << "-" if options[:stdin]
         if filenames.empty?
-          raise Thor::RequiredArgumentMissingError, 'At least one of --filenames or --stdin must be set'
+          raise(Thor::RequiredArgumentMissingError, '--filenames must be set and not empty')
         end
 
         ::Krane::OptionsHelper.with_processed_template_paths(filenames, render_erb: true) do |paths|

data/lib/krane/cluster_resource_discovery.rb

@@ -37,7 +37,7 @@ module Krane
     def fetch_resources(namespaced: false)
       command = %w(api-resources)
       command << "--namespaced=#{namespaced}"
-      raw, _, st = kubectl.run(*command, output: "wide", attempts: 5,
+      raw, err, st = kubectl.run(*command, output: "wide", attempts: 5,
         use_namespace: false)
       if st.success?
         rows = raw.split("\n")
@@ -59,7 +59,7 @@ module Krane
           resource
         end
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving api-resources: #{err}"
       end
     end
 
@@ -68,7 +68,7 @@ module Krane
     # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
     # A kind may not exist in all versions of the group.
     def fetch_api_versions
-      raw, _, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
+      raw, err, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
       # The "core" group is represented by an empty string
       versions = { "" => %w(v1) }
       if st.success?
@@ -78,6 +78,8 @@ module Krane
           versions[group] ||= []
           versions[group] << version
         end
+      else
+        raise FatalKubeAPIError, "Error retrieving api-versions: #{err}"
       end
       versions
     end
@@ -85,7 +87,8 @@ module Krane
     def version_for_kind(versions, kind)
       # Override list for kinds that don't appear in the lastest version of a group
       version_override = { "CronJob" => "v1beta1", "VolumeAttachment" => "v1beta1",
-                           "CSIDriver" => "v1beta1", "Ingress" => "v1beta1", "CSINode" => "v1beta1" }
+                           "CSIDriver" => "v1beta1", "Ingress" => "v1beta1",
+                           "CSINode" => "v1beta1", "Job" => "v1" }
 
       pattern = /v(?<major>\d+)(?<pre>alpha|beta)?(?<minor>\d+)?/
       latest = versions.sort_by do |version|
@@ -97,12 +100,12 @@ module Krane
     end
 
     def fetch_crds
-      raw_json, _, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
+      raw_json, err, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
         use_namespace: false)
       if st.success?
         JSON.parse(raw_json)["items"]
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving CustomResourceDefinition: #{err}"
       end
     end
 

data/lib/krane/concurrency.rb

@@ -3,11 +3,11 @@ module Krane
   module Concurrency
     MAX_THREADS = 8
 
-    def self.split_across_threads(all_work, &block)
+    def self.split_across_threads(all_work, max_threads: MAX_THREADS, &block)
       return if all_work.empty?
       raise ArgumentError, "Block of work is required" unless block_given?
 
-      slice_size = ((all_work.length + MAX_THREADS - 1) / MAX_THREADS)
+      slice_size = ((all_work.length + max_threads - 1) / max_threads)
       threads = []
       all_work.each_slice(slice_size) do |work_group|
         threads << Thread.new { work_group.each(&block) }

data/lib/krane/container_logs.rb

@@ -3,7 +3,7 @@ module Krane
   class ContainerLogs
     attr_reader :lines, :container_name
 
-    DEFAULT_LINE_LIMIT = 250
+    DEFAULT_LINE_LIMIT = 25
 
     def initialize(parent_id:, container_name:, namespace:, context:, logger:)
       @parent_id = parent_id

data/lib/krane/container_overrides.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+module Krane
+  class ContainerOverrides
+    attr_reader :command, :arguments, :env_vars, :image_tag
+
+    def initialize(command: nil, arguments: nil, env_vars: [], image_tag: nil)
+      @command = command
+      @arguments = arguments
+      @env_vars = env_vars
+      @image_tag = image_tag
+    end
+
+    def apply!(container)
+      container.command = command if command
+      container.args = arguments if arguments
+
+      if image_tag
+        image = container.image
+        base_image, _old_tag = image.split(':')
+        new_image = "#{base_image}:#{image_tag}"
+
+        container.image = new_image
+      end
+
+      env_args = env_vars.map do |env|
+        key, value = env.split('=', 2)
+        { name: key, value: value }
+      end
+      container.env ||= []
+      container.env = container.env.map(&:to_h) + env_args
+    end
+  end
+end

data/lib/krane/deploy_task.rb

@@ -10,7 +10,6 @@ require 'krane/resource_cache'
 require 'krane/kubernetes_resource'
 %w(
   custom_resource
-  cloudsql
   config_map
   deployment
   ingress
@@ -60,14 +59,14 @@ module Krane
       before_crs = %w(
         ResourceQuota
         NetworkPolicy
-      )
-      after_crs = %w(
         ConfigMap
         PersistentVolumeClaim
         ServiceAccount
         Role
         RoleBinding
         Secret
+      )
+      after_crs = %w(
         Pod
       )
 
@@ -117,8 +116,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue FatalDeploymentError
       false
@@ -216,6 +215,7 @@ module Krane
 
     def check_initial_status(resources)
       cache = ResourceCache.new(@task_config)
+      cache.prewarm(resources)
       Krane::Concurrency.split_across_threads(resources) { |r| r.sync(cache) }
       resources.each { |r| @logger.info(r.pretty_status) }
     end

data/lib/krane/ejson_secret_provisioner.rb

@@ -134,10 +134,13 @@ module Krane
     end
 
     def decrypt_ejson(key_dir)
-      # ejson seems to dump both errors and output to STDOUT
-      out_err, st = Open3.capture2e("EJSON_KEYDIR=#{key_dir} ejson decrypt #{@ejson_file}")
-      raise EjsonSecretError, out_err unless st.success?
-      JSON.parse(out_err)
+      out, err, st = Open3.capture3("EJSON_KEYDIR=#{key_dir} ejson decrypt #{@ejson_file}")
+      unless st.success?
+        # older ejson versions dump some errors to STDOUT
+        msg = err.presence || out
+        raise EjsonSecretError, msg
+      end
+      JSON.parse(out)
     rescue JSON::ParserError
       raise EjsonSecretError, "Failed to parse decrypted ejson"
     end

data/lib/krane/global_deploy_task.rb

@@ -46,8 +46,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue FatalDeploymentError
       false
@@ -202,6 +202,7 @@ module Krane
 
     def check_initial_status(resources)
       cache = ResourceCache.new(@task_config)
+      cache.prewarm(resources)
       Concurrency.split_across_threads(resources) { |r| r.sync(cache) }
       resources.each { |r| logger.info(r.pretty_status) }
     end

data/lib/krane/kubectl.rb

@@ -6,6 +6,7 @@ module Krane
     ERROR_MATCHERS = {
       not_found: /NotFound/,
       client_timeout: /Client\.Timeout exceeded while awaiting headers/,
+      empty: /\A\z/,
     }
     DEFAULT_TIMEOUT = 15
     MAX_RETRY_DELAY = 16
@@ -34,7 +35,16 @@ module Krane
       (1..attempts).to_a.each do |current_attempt|
         logger.debug("Running command (attempt #{current_attempt}): #{cmd.join(' ')}")
         out, err, st = Open3.capture3(*cmd)
-        logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) unless output_is_sensitive
+
+        # https://github.com/Shopify/krane/issues/395
+        unless out.valid_encoding?
+          out = out.dup.force_encoding(Encoding::UTF_8)
+        end
+
+        if logger.debug? && !output_is_sensitive
+          # don't do the gsub unless we're going to print this
+          logger.debug("Kubectl out: " + out.gsub(/\s+/, ' '))
+        end
 
         break if st.success?
         raise(ResourceNotFoundError, err) if err.match(ERROR_MATCHERS[:not_found]) && raise_if_not_found

data/lib/krane/kubernetes_resource.rb

@@ -16,7 +16,7 @@ module Krane
     TIMEOUT = 5.minutes
     LOG_LINE_COUNT = 250
     SERVER_DRY_RUN_DISABLED_ERROR =
-      /(unknown flag: --server-dry-run)|(doesn't support dry-run)|(dryRun alpha feature is disabled)/
+      /(unknown flag: --server-dry-run)|(does[\s\']n[o|']t support dry[-\s]run)|(dryRun alpha feature is disabled)/
 
     DISABLE_FETCHING_LOG_INFO = 'DISABLE_FETCHING_LOG_INFO'
     DISABLE_FETCHING_EVENT_INFO = 'DISABLE_FETCHING_EVENT_INFO'
@@ -38,6 +38,7 @@ module Krane
     LAST_APPLIED_ANNOTATION = "kubectl.kubernetes.io/last-applied-configuration"
     SENSITIVE_TEMPLATE_CONTENT = false
     SERVER_DRY_RUNNABLE = false
+    SYNC_DEPENDENCIES = []
 
     class << self
       def build(namespace: nil, context:, definition:, logger:, statsd_tags:, crd: nil, global_names: [])
@@ -59,8 +60,8 @@ module Krane
       end
 
       def class_for_kind(kind)
-        if Krane.const_defined?(kind)
-          Krane.const_get(kind)
+        if Krane.const_defined?(kind, false)
+          Krane.const_get(kind, false)
         end
       rescue NameError
         nil
@@ -255,7 +256,7 @@ module Krane
       if cause == :gave_up
         debug_heading = ColorizedString.new("#{id}: GLOBAL WATCH TIMEOUT (#{info_hash[:timeout]} seconds)").yellow
         helpful_info << "If you expected it to take longer than #{info_hash[:timeout]} seconds for your deploy"\
-        " to roll out, increase --max-watch-seconds."
+        " to roll out, increase --global-timeout."
       elsif deploy_failed?
         debug_heading = ColorizedString.new("#{id}: FAILED").red
         helpful_info << failure_message if failure_message.present?
@@ -571,7 +572,7 @@ module Krane
     def validate_with_server_side_dry_run(kubectl)
       command = ["apply", "-f", file_path, "--server-dry-run", "--output=name"]
       kubectl.run(*command, log_failure: false, output_is_sensitive: sensitive_template_content?,
-        retry_whitelist: [:client_timeout], attempts: 3)
+        retry_whitelist: [:client_timeout, :empty], attempts: 3)
     end
 
     # Local dry run is supported on only create and apply
@@ -581,7 +582,7 @@ module Krane
       verb = deploy_method == :apply ? "apply" : "create"
       command = [verb, "-f", file_path, "--dry-run", "--output=name"]
       kubectl.run(*command, log_failure: false, output_is_sensitive: sensitive_template_content?,
-        retry_whitelist: [:client_timeout], attempts: 3, use_namespace: !global?)
+        retry_whitelist: [:client_timeout, :empty], attempts: 3, use_namespace: !global?)
     end
 
     def labels

data/lib/krane/kubernetes_resource/custom_resource.rb

@@ -62,7 +62,7 @@ module Krane
       kind
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       @crd.validate_rollout_conditions

data/lib/krane/kubernetes_resource/custom_resource_definition.rb

@@ -66,7 +66,7 @@ module Krane
       @rollout_conditions = nil
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       validate_rollout_conditions

data/lib/krane/kubernetes_resource/daemon_set.rb

@@ -3,6 +3,7 @@ require "krane/kubernetes_resource/pod_set_base"
 module Krane
   class DaemonSet < PodSetBase
     TIMEOUT = 5.minutes
+    SYNC_DEPENDENCIES = %w(Pod)
     attr_reader :pods
 
     def sync(cache)

data/lib/krane/kubernetes_resource/deployment.rb

@@ -4,6 +4,7 @@ require 'krane/kubernetes_resource/replica_set'
 module Krane
   class Deployment < KubernetesResource
     TIMEOUT = 7.minutes
+    SYNC_DEPENDENCIES = %w(Pod ReplicaSet)
     REQUIRED_ROLLOUT_ANNOTATION_SUFFIX = "required-rollout"
     REQUIRED_ROLLOUT_ANNOTATION_DEPRECATED = "kubernetes-deploy.shopify.io/#{REQUIRED_ROLLOUT_ANNOTATION_SUFFIX}"
     REQUIRED_ROLLOUT_ANNOTATION = "krane.shopify.io/#{REQUIRED_ROLLOUT_ANNOTATION_SUFFIX}"
@@ -96,7 +97,7 @@ module Krane
       progress_condition.present? ? deploy_failing_to_progress? : super
     end
 
-    def validate_definition(*)
+    def validate_definition(*, **)
       super
 
       unless REQUIRED_ROLLOUT_TYPES.include?(required_rollout) || percent?(required_rollout)
@@ -190,7 +191,7 @@ module Krane
     def min_available_replicas
       if percent?(required_rollout)
         (desired_replicas * required_rollout.to_i / 100.0).ceil
-      elsif max_unavailable =~ /%/
+      elsif max_unavailable.is_a?(String) && max_unavailable =~ /%/
         (desired_replicas * (100 - max_unavailable.to_i) / 100.0).ceil
       else
         desired_replicas - max_unavailable.to_i

data/lib/krane/kubernetes_resource/pod.rb

@@ -219,14 +219,7 @@ module Krane
         limbo_reason = @status.dig("state", "waiting", "reason")
         limbo_message = @status.dig("state", "waiting", "message")
 
-        if @status.dig("lastState", "terminated", "reason") == "ContainerCannotRun"
-          # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
-          exit_code = @status.dig('lastState', 'terminated', 'exitCode')
-          "Failed to start (exit #{exit_code}): #{@status.dig('lastState', 'terminated', 'message')}"
-        elsif @status.dig("state", "terminated", "reason") == "ContainerCannotRun"
-          exit_code = @status.dig('state', 'terminated', 'exitCode')
-          "Failed to start (exit #{exit_code}): #{@status.dig('state', 'terminated', 'message')}"
-        elsif limbo_reason == "CrashLoopBackOff"
+        if limbo_reason == "CrashLoopBackOff"
           exit_code = @status.dig('lastState', 'terminated', 'exitCode')
           "Crashing repeatedly (exit #{exit_code}). See logs for more information."
         elsif limbo_reason == "ErrImagePull" && limbo_message.match(/not found/i)
@@ -234,6 +227,17 @@ module Krane
           "Did you wait for it to be built and pushed to the registry before deploying?"
         elsif limbo_reason == "CreateContainerConfigError"
           "Failed to generate container configuration: #{limbo_message}"
+        elsif @status.dig("lastState", "terminated", "reason") == "ContainerCannotRun"
+          # ref: https://github.com/kubernetes/kubernetes/blob/562e721ece8a16e05c7e7d6bdd6334c910733ab2/pkg/kubelet/dockershim/docker_container.go#L353
+          exit_code = @status.dig('lastState', 'terminated', 'exitCode')
+          # We've observed failures here that are actually issues with the node or kube infra, and not with the
+          # container. These issues have been transient and result in a 128 exit code, so do not treat these as fatal.
+          return if exit_code == 128
+          "Failed to start (exit #{exit_code}): #{@status.dig('lastState', 'terminated', 'message')}"
+        elsif @status.dig("state", "terminated", "reason") == "ContainerCannotRun"
+          exit_code = @status.dig('state', 'terminated', 'exitCode')
+          return if exit_code == 128
+          "Failed to start (exit #{exit_code}): #{@status.dig('state', 'terminated', 'message')}"
         end
       end
 

data/lib/krane/kubernetes_resource/replica_set.rb

@@ -4,6 +4,7 @@ require 'krane/kubernetes_resource/pod_set_base'
 module Krane
   class ReplicaSet < PodSetBase
     TIMEOUT = 5.minutes
+    SYNC_DEPENDENCIES = %w(Pod)
     attr_reader :pods
 
     def initialize(namespace:, context:, definition:, logger:, statsd_tags: nil,
@@ -17,7 +18,7 @@ module Krane
 
     def sync(cache)
       super
-      @pods = fetch_pods_if_needed(cache) || []
+      @pods = exists? ? find_pods(cache) : []
     end
 
     def status
@@ -58,21 +59,6 @@ module Krane
       observed_generation != current_generation
     end
 
-    def fetch_pods_if_needed(cache)
-      # If the ReplicaSet doesn't exist, its pods won't either
-      return unless exists?
-      # If the status hasn't been updated yet, we're not going to make a determination anyway
-      return if stale_status?
-      # If we don't want any pods at all, we don't need to look for them
-      return if desired_replicas == 0
-      # We only need to fetch pods so that deploy_failed? can check that they aren't ALL bad.
-      # If we can already tell some pods are ok from the RS data, don't bother fetching them (which can be expensive)
-      # Lower numbers here make us more susceptible to being fooled by replicas without probes briefly appearing ready
-      return if ready_replicas > 1
-
-      find_pods(cache)
-    end
-
     def rollout_data
       return { "replicas" => 0 } unless exists?
       { "replicas" => 0 }.merge(