kubernetes-deploy 0.22.0 → 0.23.0

data/lib/kubernetes-deploy/errors.rb

@@ -3,6 +3,7 @@ module KubernetesDeploy
   class FatalDeploymentError < StandardError; end
   class FatalKubeAPIError < FatalDeploymentError; end
   class KubectlError < StandardError; end
+  class TaskConfigurationError < FatalDeploymentError; end
 
   class InvalidTemplateError < FatalDeploymentError
     attr_reader :content

data/lib/kubernetes-deploy/formatted_logger.rb

@@ -6,12 +6,26 @@ module KubernetesDeploy
   class FormattedLogger < Logger
     include DeferredSummaryLogging
 
-    def self.build(namespace, context, stream = $stderr, verbose_prefix: false)
+    def self.indent_four(str)
+      "    " + str.to_s.gsub("\n", "\n    ")
+    end
+
+    def self.build(namespace = nil, context = nil, stream = $stderr, verbose_prefix: false)
       l = new(stream)
       l.level = level_from_env
 
+      middle = if verbose_prefix
+        if namespace.blank?
+          raise ArgumentError, 'Must pass a namespace if logging verbosely'
+        end
+        if context.blank?
+          raise ArgumentError, 'Must pass a context if logging verbosely'
+        end
+
+        "[#{context}][#{namespace}]"
+      end
+
       l.formatter = proc do |severity, datetime, _progname, msg|
-        middle = verbose_prefix ? "[#{context}][#{namespace}]" : ""
         colorized_line = ColorizedString.new("[#{severity}][#{datetime}]#{middle}\t#{msg}\n")
 
         case severity

data/lib/kubernetes-deploy/kubeclient_builder/google_friendly_config.rb

@@ -32,12 +32,10 @@ module KubernetesDeploy
       private
 
       def json_error_message(body)
-        json_error_msg = begin
-          JSON.parse(body || '') || {}
-        rescue JSON::ParserError
-          {}
-        end
-        json_error_msg['message']
+        err = JSON.parse(body || '') || {}
+        err['message']
+      rescue JSON::ParserError
+        nil
       end
     end
   end

data/lib/kubernetes-deploy/kubectl.rb

@@ -2,7 +2,7 @@
 
 module KubernetesDeploy
   class Kubectl
-    DEFAULT_TIMEOUT = 30
+    DEFAULT_TIMEOUT = 15
     NOT_FOUND_ERROR_TEXT = 'NotFound'
 
     class ResourceNotFoundError < StandardError; end
@@ -20,32 +20,43 @@ module KubernetesDeploy
       raise ArgumentError, "context is required" if context.blank?
     end
 
-    def run(*args, log_failure: nil, use_context: true, use_namespace: true, raise_if_not_found: false)
+    def run(*args, log_failure: nil, use_context: true, use_namespace: true, raise_if_not_found: false, attempts: 1)
       log_failure = @log_failure_by_default if log_failure.nil?
 
       args = args.unshift("kubectl")
       args.push("--namespace=#{@namespace}") if use_namespace
       args.push("--context=#{@context}")     if use_context
       args.push("--request-timeout=#{@default_timeout}") if @default_timeout
+      out, err, st = nil
 
-      @logger.debug Shellwords.join(args)
-      out, err, st = Open3.capture3(*args)
-      @logger.debug(out.shellescape) unless output_is_sensitive?
+      (1..attempts).to_a.each do |attempt|
+        @logger.debug "Running command (attempt #{attempt}): #{args.join(' ')}"
+        out, err, st = Open3.capture3(*args)
+        @logger.debug("Kubectl out: " + out.gsub(/\s+/, ' ')) unless output_is_sensitive?
+
+        break if st.success?
 
-      unless st.success?
         if log_failure
-          @logger.warn("The following command failed: #{Shellwords.join(args)}")
+          @logger.warn("The following command failed (attempt #{attempt}/#{attempts}): #{Shellwords.join(args)}")
           @logger.warn(err) unless output_is_sensitive?
         end
 
-        if raise_if_not_found && err.match(NOT_FOUND_ERROR_TEXT)
-          raise ResourceNotFoundError, err
+        if err.match(NOT_FOUND_ERROR_TEXT)
+          raise(ResourceNotFoundError, err) if raise_if_not_found
+        else
+          @logger.debug("Kubectl err: #{err}") unless output_is_sensitive?
+          StatsD.increment('kubectl.error', 1, tags: { context: @context, namespace: @namespace, cmd: args[1] })
         end
+        sleep retry_delay(attempt) unless attempt == attempts
       end
 
       [out.chomp, err.chomp, st]
     end
 
+    def retry_delay(attempt)
+      attempt
+    end
+
     def version_info
       @version_info ||=
         begin

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -121,8 +121,8 @@ module KubernetesDeploy
       file.path
     end
 
-    def sync(mediator)
-      @instance_data = mediator.get_instance(kubectl_resource_type, name, raise_if_not_found: true)
+    def sync(cache)
+      @instance_data = cache.get_instance(kubectl_resource_type, name, raise_if_not_found: true)
     rescue KubernetesDeploy::Kubectl::ResourceNotFoundError
       @disappeared = true if deploy_started?
       @instance_data = {}
@@ -195,7 +195,7 @@ module KubernetesDeploy
 
     def sync_debug_info(kubectl)
       @debug_events = fetch_events(kubectl) unless ENV[DISABLE_FETCHING_EVENT_INFO]
-      @debug_logs = fetch_debug_logs(kubectl) if print_debug_logs? && !ENV[DISABLE_FETCHING_LOG_INFO]
+      @debug_logs = fetch_debug_logs if print_debug_logs? && !ENV[DISABLE_FETCHING_LOG_INFO]
     end
 
     def debug_message(cause = nil, info_hash = {})
@@ -293,7 +293,7 @@ module KubernetesDeploy
 
     def report_status_to_statsd(watch_time)
       unless @statsd_report_done
-        ::StatsD.measure('resource.duration', watch_time, tags: statsd_tags)
+        StatsD.distribution('resource.duration', watch_time, tags: statsd_tags)
         @statsd_report_done = true
       end
     end
@@ -407,8 +407,7 @@ module KubernetesDeploy
       else
         "unknown"
       end
-      tags = %W(context:#{context} namespace:#{namespace} resource:#{id}
-                type:#{type} sha:#{ENV['REVISION']} status:#{status})
+      tags = %W(context:#{context} namespace:#{namespace} type:#{type} status:#{status})
       tags | @optional_statsd_tags
     end
   end

data/lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb

@@ -3,11 +3,10 @@ module KubernetesDeploy
   class Cloudsql < KubernetesResource
     TIMEOUT = 10.minutes
 
-    SYNC_DEPENDENCIES = %w(Deployment Service)
-    def sync(mediator)
+    def sync(cache)
       super
-      @proxy_deployment = mediator.get_instance(Deployment.kind, "cloudsql-#{cloudsql_resource_uuid}")
-      @proxy_service = mediator.get_instance(Service.kind, "cloudsql-#{@name}")
+      @proxy_deployment = cache.get_instance(Deployment.kind, "cloudsql-#{cloudsql_resource_uuid}")
+      @proxy_service = cache.get_instance(Service.kind, "cloudsql-#{@name}")
     end
 
     def status

data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb

@@ -5,10 +5,9 @@ module KubernetesDeploy
     TIMEOUT = 5.minutes
     attr_reader :pods
 
-    SYNC_DEPENDENCIES = %w(Pod)
-    def sync(mediator)
+    def sync(cache)
       super
-      @pods = exists? ? find_pods(mediator) : []
+      @pods = exists? ? find_pods(cache) : []
     end
 
     def status
@@ -28,9 +27,9 @@ module KubernetesDeploy
       observed_generation == current_generation
     end
 
-    def fetch_debug_logs(kubectl)
+    def fetch_debug_logs
       most_useful_pod = pods.find(&:deploy_failed?) || pods.find(&:deploy_timed_out?) || pods.first
-      most_useful_pod.fetch_debug_logs(kubectl)
+      most_useful_pod.fetch_debug_logs
     end
 
     def print_debug_logs?

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -6,10 +6,9 @@ module KubernetesDeploy
     REQUIRED_ROLLOUT_TYPES = %w(maxUnavailable full none).freeze
     DEFAULT_REQUIRED_ROLLOUT = 'full'
 
-    SYNC_DEPENDENCIES = %w(Pod ReplicaSet)
-    def sync(mediator)
+    def sync(cache)
       super
-      @latest_rs = exists? ? find_latest_rs(mediator) : nil
+      @latest_rs = exists? ? find_latest_rs(cache) : nil
     end
 
     def status
@@ -27,8 +26,8 @@ module KubernetesDeploy
       @latest_rs.present?
     end
 
-    def fetch_debug_logs(kubectl)
-      @latest_rs.fetch_debug_logs(kubectl)
+    def fetch_debug_logs
+      @latest_rs.fetch_debug_logs
     end
 
     def deploy_succeeded?
@@ -151,8 +150,8 @@ module KubernetesDeploy
       progress_condition["status"] == 'False'
     end
 
-    def find_latest_rs(mediator)
-      all_rs_data = mediator.get_all(ReplicaSet.kind, @instance_data["spec"]["selector"]["matchLabels"])
+    def find_latest_rs(cache)
+      all_rs_data = cache.get_all(ReplicaSet.kind, @instance_data["spec"]["selector"]["matchLabels"])
       current_revision = @instance_data["metadata"]["annotations"]["deployment.kubernetes.io/revision"]
 
       latest_rs_data = all_rs_data.find do |rs|
@@ -170,7 +169,7 @@ module KubernetesDeploy
         parent: "#{@name.capitalize} deployment",
         deploy_started_at: @deploy_started_at
       )
-      rs.sync(mediator)
+      rs.sync(cache)
       rs
     end
 

data/lib/kubernetes-deploy/kubernetes_resource/memcached.rb

@@ -4,12 +4,11 @@ module KubernetesDeploy
     TIMEOUT = 5.minutes
     CONFIGMAP_NAME = "memcached-url"
 
-    SYNC_DEPENDENCIES = %w(Deployment Service ConfigMap)
-    def sync(mediator)
+    def sync(cache)
       super
-      @deployment = mediator.get_instance(Deployment.kind, "memcached-#{@name}")
-      @service = mediator.get_instance(Service.kind, "memcached-#{@name}")
-      @configmap = mediator.get_instance(ConfigMap.kind, CONFIGMAP_NAME)
+      @deployment = cache.get_instance(Deployment.kind, "memcached-#{@name}")
+      @service = cache.get_instance(Service.kind, "memcached-#{@name}")
+      @configmap = cache.get_instance(ConfigMap.kind, CONFIGMAP_NAME)
     end
 
     def status

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -25,12 +25,12 @@ module KubernetesDeploy
             logger: logger, statsd_tags: statsd_tags)
     end
 
-    def sync(mediator)
+    def sync(_cache)
       super
       raise_predates_deploy_error if exists? && unmanaged? && !deploy_started?
 
       if exists?
-        logs.sync(mediator.kubectl) if unmanaged?
+        logs.sync if unmanaged?
         update_container_statuses(@instance_data["status"])
       else # reset
         @containers.each(&:reset_status)
@@ -85,8 +85,8 @@ module KubernetesDeploy
       "#{phase_failure_message} #{container_problems}".strip.presence
     end
 
-    def fetch_debug_logs(kubectl)
-      logs.sync(kubectl)
+    def fetch_debug_logs
+      logs.sync
       logs
     end
 
@@ -123,7 +123,9 @@ module KubernetesDeploy
       @logs ||= KubernetesDeploy::RemoteLogs.new(
         logger: @logger,
         parent_id: id,
-        container_names: @containers.map(&:name)
+        container_names: @containers.map(&:name),
+        namespace: @namespace,
+        context: @context
       )
     end
 

data/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb

@@ -16,9 +16,15 @@ module KubernetesDeploy
       own_events.merge(most_useful_pod.fetch_events(kubectl))
     end
 
-    def fetch_debug_logs(kubectl)
-      logs = KubernetesDeploy::RemoteLogs.new(logger: @logger, parent_id: id, container_names: container_names)
-      logs.sync(kubectl)
+    def fetch_debug_logs
+      logs = KubernetesDeploy::RemoteLogs.new(
+        logger: @logger,
+        parent_id: id,
+        container_names: container_names,
+        namespace: @namespace,
+        context: @context
+      )
+      logs.sync
       logs
     end
 
@@ -42,8 +48,8 @@ module KubernetesDeploy
       regular_containers + init_containers
     end
 
-    def find_pods(mediator)
-      all_pods = mediator.get_all(Pod.kind, @instance_data["spec"]["selector"]["matchLabels"])
+    def find_pods(cache)
+      all_pods = cache.get_all(Pod.kind, @instance_data["spec"]["selector"]["matchLabels"])
 
       all_pods.each_with_object([]) do |pod_data, relevant_pods|
         next unless parent_of_pod?(pod_data)
@@ -55,7 +61,7 @@ module KubernetesDeploy
           parent: "#{name.capitalize} #{type}",
           deploy_started_at: @deploy_started_at
         )
-        pod.sync(mediator)
+        pod.sync(cache)
         relevant_pods << pod
       end
     end

data/lib/kubernetes-deploy/kubernetes_resource/redis.rb

@@ -4,15 +4,14 @@ module KubernetesDeploy
     TIMEOUT = 5.minutes
     UUID_ANNOTATION = "redis.stable.shopify.io/owner_uid"
 
-    SYNC_DEPENDENCIES = %w(Deployment Service)
-    def sync(mediator)
+    def sync(cache)
       super
 
-      @deployment = mediator.get_instance(Deployment.kind, name)
-      @deployment = mediator.get_instance(Deployment.kind, deprecated_name) if @deployment.empty?
+      @deployment = cache.get_instance(Deployment.kind, name)
+      @deployment = cache.get_instance(Deployment.kind, deprecated_name) if @deployment.empty?
 
-      @service = mediator.get_instance(Service.kind, name)
-      @service = mediator.get_instance(Service.kind, deprecated_name) if @service.empty?
+      @service = cache.get_instance(Service.kind, name)
+      @service = cache.get_instance(Service.kind, deprecated_name) if @service.empty?
     end
 
     def status

data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb

@@ -14,10 +14,9 @@ module KubernetesDeploy
             logger: logger, statsd_tags: statsd_tags)
     end
 
-    SYNC_DEPENDENCIES = %w(Pod)
-    def sync(mediator)
+    def sync(cache)
       super
-      @pods = exists? ? find_pods(mediator) : []
+      @pods = fetch_pods_if_needed(cache) || []
     end
 
     def status
@@ -26,7 +25,7 @@ module KubernetesDeploy
     end
 
     def deploy_succeeded?
-      observed_generation == current_generation &&
+      return false if stale_status?
       desired_replicas == rollout_data["availableReplicas"].to_i &&
       desired_replicas == rollout_data["readyReplicas"].to_i
     end
@@ -34,7 +33,7 @@ module KubernetesDeploy
     def deploy_failed?
       pods.present? &&
       pods.all?(&:deploy_failed?) &&
-      observed_generation == current_generation
+      !stale_status?
     end
 
     def desired_replicas
@@ -54,6 +53,25 @@ module KubernetesDeploy
 
     private
 
+    def stale_status?
+      observed_generation != current_generation
+    end
+
+    def fetch_pods_if_needed(cache)
+      # If the ReplicaSet doesn't exist, its pods won't either
+      return unless exists?
+      # If the status hasn't been updated yet, we're not going to make a determination anyway
+      return if stale_status?
+      # If we don't want any pods at all, we don't need to look for them
+      return if desired_replicas == 0
+      # We only need to fetch pods so that deploy_failed? can check that they aren't ALL bad.
+      # If we can already tell some pods are ok from the RS data, don't bother fetching them (which can be expensive)
+      # Lower numbers here make us more susceptible to being fooled by replicas without probes briefly appearing ready
+      return if ready_replicas > 1
+
+      find_pods(cache)
+    end
+
     def rollout_data
       return { "replicas" => 0 } unless exists?
       { "replicas" => 0 }.merge(

data/lib/kubernetes-deploy/kubernetes_resource/role.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+module KubernetesDeploy
+  class Role < KubernetesResource
+    TIMEOUT = 30.seconds
+
+    def status
+      exists? ? "Created" : "Unknown"
+    end
+
+    def deploy_succeeded?
+      exists?
+    end
+
+    def deploy_failed?
+      false
+    end
+
+    def timeout_message
+      UNUSUAL_FAILURE_MESSAGE
+    end
+  end
+end

data/lib/kubernetes-deploy/kubernetes_resource/service.rb

@@ -3,11 +3,15 @@ module KubernetesDeploy
   class Service < KubernetesResource
     TIMEOUT = 7.minutes
 
-    SYNC_DEPENDENCIES = %w(Pod Deployment)
-    def sync(mediator)
+    def sync(cache)
       super
-      @related_deployments = selector.present? ? mediator.get_all(Deployment.kind, selector) : []
-      @related_pods = selector.present? ? mediator.get_all(Pod.kind, selector) : []
+      if exists? && selector.present?
+        @related_deployments = cache.get_all(Deployment.kind, selector)
+        @related_pods = cache.get_all(Pod.kind, selector)
+      else
+        @related_deployments = []
+        @related_pods = []
+      end
     end
 
     def status