kubernetes-deploy 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f9ccce3d57089a13afd20e62c5bba0b49c08b92
-  data.tar.gz: 58f63b9dbda402ec6cee2833ec0beb209aa3d51c
+  metadata.gz: b83794f813d2a4f1b2b3aba98a9181fc2c041f85
+  data.tar.gz: 1d0dc0a38cdd5a44adb1e914751cea41dd8dbce1
 SHA512:
-  metadata.gz: 7969d60b7979ef48b82cb717a22a153203ce495bd8c94c6c8fb1a0c711ed229b7bb32de359b7f71d0b812c64c53caf7afaafab362295bcf8ace6bc090649ef7b
-  data.tar.gz: 214dfbc251bcb0a121ec49ca8e37a803793e8f6aa7bcc1fdac8b5069d149abbdd0154b24f58324e5c9eb3c4cb6e43b705946862db5963b523887285aeb5d31ad
+  metadata.gz: 82153cb31bd8108540cdbe92fe25d25e6c4d09e3f9781c00f74f664ca2ea1a5fa65311bd940565ea5b7fefa9ec4bcae36f533855cc6f068b4970c3ecf9e06478
+  data.tar.gz: 30fbefaae8d80f503ec2e31664a4ee29097ee80a92f9d398d6ce4e1e76d0f350855fd85048ae6ae7d1fb8932abf6be48b754d4e569626ae0a52e3fb716f54e6b

data/.buildkite/pipeline.nightly.yml

@@ -0,0 +1,28 @@
+  - name: 'Run Test Suite (:kubernetes: 1.9-latest)'
+    command: bin/ci
+    agents:
+      queue: minikube-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.9-latest
+  - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
+    command: bin/ci
+    agents:
+      queue: minikube-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.8-latest
+  - name: 'Run Test Suite (:kubernetes: 1.7-latest)'
+    command: bin/ci
+    agents:
+      queue: minikube-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.7-latest
+  - name: 'Run Test Suite (:kubernetes: 1.6.4)'
+    command: bin/ci
+    agents:
+      queue: minikube-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.6.4

data/.buildkite/pipeline.yml

@@ -1,8 +1,21 @@
-  - name: 'Run Test Suite (:kubernetes: 1.7.5)'
+  - name: 'Run Test Suite (:kubernetes: 1.8-latest)'
     command: bin/ci
     agents:
-      queue: pipa-production-minikube-v1.7.5
+      queue: minikube-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.8-latest
+  - name: 'Run Test Suite (:kubernetes: 1.7-latest)'
+    command: bin/ci
+    agents:
+      queue: minikube-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.7-latest
   - name: 'Run Test Suite (:kubernetes: 1.6.4)'
     command: bin/ci
     agents:
-      queue: pipa-production-minikube-v1.6.4
+      queue: minikube-ci
+    env:
+      LOGGING_LEVEL: 4
+      KUBERNETES_VERSION: v1.6.4

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### 0.14.0
+*Bug Fixes*
+- Fix incorrect timeouts occasionally observed on deployments using progressDeadlineSeconds in Kubernetes <1.7.7
+
+*Enhancements*
+- Renamed `KubernetesDeploy::Runner` (which powers `exe/kubernetes-deploy`) to `KubernetesDeploy::DeployTask`. This increases consistency between our primary class names and avoids confusion with `KubernetesDeploy::RunnerTask` (which powers `exe/kubernetes-run`).
+- Improved output related to timeouts. For deployments, both failure and timeout output now mentions the referenced replica set.
+- Small improvements to the reliability of the success polling.
+- EjsonSecretProvisioner no longer logs kubectl command output (which may contain secret data) when debug-level logging is enabled.
+
 ### 0.13.0
 *Features*
 - Added support for StatefulSets for kubernetes 1.7+ using RollingUpdate

data/bin/ci

@@ -6,11 +6,11 @@ if [[ -n "${DEBUG:+set}" ]]; then
 fi
 
 docker run --rm \
-	--net=host \
-	-v "$HOME/.kube":/"$HOME/.kube" \
-    -v "$HOME/.minikube":/"$HOME/.minikube" \
+    --net=host \
+    -v "$HOME/.kube":"/root/.kube" \
+    -v "$HOME/.minikube":"$HOME/.minikube" \
     -v "$PWD":/usr/src/app \
-    -v "/usr/bin/minikube/kubectl":"/usr/bin/kubectl" \
+    -v "/usr/bin/kubectl":"/usr/bin/kubectl" \
     -e CI=1 \
     -e CODECOV_TOKEN=$CODECOV_TOKEN \
     -e COVERAGE=1 \

data/exe/kubernetes-deploy

@@ -22,7 +22,7 @@ ARGV.options do |opts|
   end
 
   opts.on("--skip-wait", "Skip verification of non-priority-resource success (not recommended)") { skip_wait = true }
-  prot_ns = KubernetesDeploy::Runner::PROTECTED_NAMESPACES.join(', ')
+  prot_ns = KubernetesDeploy::DeployTask::PROTECTED_NAMESPACES.join(', ')
   opts.on("--allow-protected-ns", "Enable deploys to #{prot_ns}; requires --no-prune") { allow_protected_ns = true }
   opts.on("--no-prune", "Disable deletion of resources that do not appear in the template dir") { prune = false }
   opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT)") { |v| template_dir = v }
@@ -58,7 +58,7 @@ namespace = ARGV[0]
 context = ARGV[1]
 logger = KubernetesDeploy::FormattedLogger.build(namespace, context, verbose_prefix: verbose_log_prefix)
 
-runner = KubernetesDeploy::Runner.new(
+runner = KubernetesDeploy::DeployTask.new(
   namespace: namespace,
   context: context,
   current_sha: revision,

data/lib/kubernetes-deploy.rb

@@ -1,4 +1,4 @@
-# rubocop:disable Style/FileName
+# rubocop:disable Naming/FileName
 # frozen_string_literal: true
 
 require 'active_support/core_ext/object/blank'
@@ -13,7 +13,7 @@ require 'colorized_string'
 require 'kubernetes-deploy/version'
 require 'kubernetes-deploy/errors'
 require 'kubernetes-deploy/formatted_logger'
-require 'kubernetes-deploy/runner'
+require 'kubernetes-deploy/deploy_task'
 require 'kubernetes-deploy/statsd'
 require 'kubernetes-deploy/concurrency'
 

data/lib/kubernetes-deploy/{runner.rb → deploy_task.rb}

@@ -37,7 +37,7 @@ require 'kubernetes-deploy/kubeclient_builder'
 require 'kubernetes-deploy/ejson_secret_provisioner'
 
 module KubernetesDeploy
-  class Runner
+  class DeployTask
     include KubeclientBuilder
 
     PREDEPLOY_SEQUENCE = %w(
@@ -326,15 +326,15 @@ module KubernetesDeploy
         @logger.info("Deploying resources:")
       else
         resource = resources.first
-        @logger.info("Deploying #{resource.id} (timeout: #{resource.timeout}s)")
+        @logger.info("Deploying #{resource.id} (#{resource.pretty_timeout_type})")
       end
 
       # Apply can be done in one large batch, the rest have to be done individually
       applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
 
       individuals.each do |r|
-        @logger.info("- #{r.id} (timeout: #{r.timeout}s)") if resources.length > 1
-        r.deploy_started = Time.now.utc
+        @logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
+        r.deploy_started_at = Time.now.utc
         case r.deploy_method
         when :replace
           _, _, replace_st = kubectl.run("replace", "-f", r.file_path, log_failure: false)
@@ -369,9 +369,9 @@ module KubernetesDeploy
 
       command = ["apply"]
       resources.each do |r|
-        @logger.info("- #{r.id} (timeout: #{r.timeout}s)") if resources.length > 1
+        @logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
         command.push("-f", r.file_path)
-        r.deploy_started = Time.now.utc
+        r.deploy_started_at = Time.now.utc
       end
 
       if prune

data/lib/kubernetes-deploy/ejson_secret_provisioner.rb

@@ -23,7 +23,13 @@ module KubernetesDeploy
       @ejson_file = "#{template_dir}/#{EJSON_SECRETS_FILE}"
       @logger = logger
       @prune = prune
-      @kubectl = Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: false)
+      @kubectl = Kubectl.new(
+        namespace: @namespace,
+        context: @context,
+        logger: @logger,
+        log_failure_by_default: false,
+        output_is_sensitive: true # output may contain ejson secrets
+      )
     end
 
     def secret_changes_required?

data/lib/kubernetes-deploy/kubectl.rb

@@ -2,12 +2,14 @@
 
 module KubernetesDeploy
   class Kubectl
-    def initialize(namespace:, context:, logger:, log_failure_by_default:, default_timeout: '30s')
+    def initialize(namespace:, context:, logger:, log_failure_by_default:, default_timeout: '30s',
+      output_is_sensitive: false)
       @namespace = namespace
       @context = context
       @logger = logger
       @log_failure_by_default = log_failure_by_default
       @default_timeout = default_timeout
+      @output_is_sensitive = output_is_sensitive
 
       raise ArgumentError, "namespace is required" if namespace.blank?
       raise ArgumentError, "context is required" if context.blank?
@@ -23,11 +25,11 @@ module KubernetesDeploy
 
       @logger.debug Shellwords.join(args)
       out, err, st = Open3.capture3(*args)
-      @logger.debug(out.shellescape)
+      @logger.debug(out.shellescape) unless output_is_sensitive?
 
       if !st.success? && log_failure
         @logger.warn("The following command failed: #{Shellwords.join(args)}")
-        @logger.warn(err)
+        @logger.warn(err) unless output_is_sensitive?
       end
       [out.chomp, err.chomp, st]
     end
@@ -51,6 +53,10 @@ module KubernetesDeploy
 
     private
 
+    def output_is_sensitive?
+      @output_is_sensitive
+    end
+
     def extract_version_info_from_kubectl_response(response)
       info = {}
       response.each_line do |l|

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -6,8 +6,8 @@ require 'kubernetes-deploy/kubectl'
 
 module KubernetesDeploy
   class KubernetesResource
-    attr_reader :name, :namespace, :file, :context, :validation_error_msg
-    attr_writer :type, :deploy_started
+    attr_reader :name, :namespace, :context, :validation_error_msg
+    attr_writer :type, :deploy_started_at
 
     TIMEOUT = 5.minutes
     LOG_LINE_COUNT = 250
@@ -42,6 +42,10 @@ module KubernetesDeploy
       self.class.timeout
     end
 
+    def pretty_timeout_type
+      "timeout: #{timeout}s"
+    end
+
     def initialize(namespace:, context:, definition:, logger:)
       # subclasses must also set these if they define their own initializer
       @name = definition.dig("metadata", "name")
@@ -86,8 +90,12 @@ module KubernetesDeploy
       false
     end
 
+    def deploy_started?
+      @deploy_started_at.present?
+    end
+
     def deploy_succeeded?
-      if @deploy_started && !@success_assumption_warning_shown
+      if deploy_started? && !@success_assumption_warning_shown
         @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.")
         @success_assumption_warning_shown = true
       end
@@ -103,16 +111,12 @@ module KubernetesDeploy
     end
 
     def type
-      @type || self.class.name.split('::').last
-    end
-
-    def deploy_finished?
-      deploy_failed? || deploy_succeeded? || deploy_timed_out?
+      @type || self.class.name.demodulize
     end
 
     def deploy_timed_out?
-      return false unless @deploy_started
-      !deploy_succeeded? && !deploy_failed? && (Time.now.utc - @deploy_started > timeout)
+      return false unless deploy_started?
+      !deploy_succeeded? && !deploy_failed? && (Time.now.utc - @deploy_started_at > timeout)
     end
 
     # Expected values: :apply, :replace, :replace_force
@@ -125,9 +129,14 @@ module KubernetesDeploy
       if deploy_failed?
         helpful_info << ColorizedString.new("#{id}: FAILED").red
         helpful_info << failure_message if failure_message.present?
-      else
-        helpful_info << ColorizedString.new("#{id}: TIMED OUT").yellow + " (limit: #{timeout}s)"
+      elsif deploy_timed_out?
+        helpful_info << ColorizedString.new("#{id}: TIMED OUT (#{pretty_timeout_type})").yellow
         helpful_info << timeout_message if timeout_message.present?
+      else
+        # Arriving in debug_message when we neither failed nor timed out is very unexpected. Dump all available info.
+        helpful_info << ColorizedString.new("#{id}: MONITORING ERROR").red
+        helpful_info << failure_message if failure_message.present?
+        helpful_info << timeout_message if timeout_message.present? && timeout_message != STANDARD_TIMEOUT_MESSAGE
       end
       helpful_info << "  - Final status: #{status}"
 
@@ -178,7 +187,7 @@ module KubernetesDeploy
 
       event_collector = Hash.new { |hash, key| hash[key] = [] }
       Event.extract_all_from_go_template_blob(out).each_with_object(event_collector) do |candidate, events|
-        events[id] << candidate.to_s if candidate.seen_since?(@deploy_started - 5.seconds)
+        events[id] << candidate.to_s if candidate.seen_since?(@deploy_started_at - 5.seconds)
       end
     end
 

data/lib/kubernetes-deploy/kubernetes_resource/bucket.rb

@@ -7,7 +7,7 @@ module KubernetesDeploy
     end
 
     def deploy_succeeded?
-      return false unless @deploy_started
+      return false unless deploy_started?
 
       unless @success_assumption_warning_shown
         @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.")

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -11,15 +11,21 @@ module KubernetesDeploy
         deployment_data = JSON.parse(raw_json)
         @desired_replicas = deployment_data["spec"]["replicas"].to_i
         @latest_rs = find_latest_rs(deployment_data)
+
         @rollout_data = { "replicas" => 0 }.merge(deployment_data["status"]
           .slice("replicas", "updatedReplicas", "availableReplicas", "unavailableReplicas"))
         @status = @rollout_data.map { |state_replicas, num| "#{num} #{state_replicas.chop.pluralize(num)}" }.join(", ")
+
         conditions = deployment_data.fetch("status", {}).fetch("conditions", [])
-        @progress = conditions.find { |condition| condition['type'] == 'Progressing' }
+        @progress_condition = conditions.find { |condition| condition['type'] == 'Progressing' }
+        @progress_deadline = deployment_data['spec']['progressDeadlineSeconds']
       else # reset
         @latest_rs = nil
         @rollout_data = { "replicas" => 0 }
         @status = nil
+        @progress_condition = nil
+        @progress_deadline = @definition['spec']['progressDeadlineSeconds']
+        @desired_replicas = -1
       end
     end
 
@@ -35,7 +41,7 @@ module KubernetesDeploy
     end
 
     def deploy_succeeded?
-      return false unless @latest_rs
+      return false unless @latest_rs.present?
 
       @latest_rs.deploy_succeeded? &&
       @latest_rs.desired_replicas == @desired_replicas && # latest RS fully scaled up
@@ -44,26 +50,31 @@ module KubernetesDeploy
     end
 
     def deploy_failed?
-      @latest_rs && @latest_rs.deploy_failed?
+      @latest_rs&.deploy_failed?
     end
 
     def failure_message
-      @latest_rs&.failure_message
+      return unless @latest_rs.present?
+      "Latest ReplicaSet: #{@latest_rs.name}\n\n#{@latest_rs.failure_message}"
     end
 
     def timeout_message
-      progress_seconds = @definition['spec']['progressDeadlineSeconds']
-      if progress_seconds
-        "Deploy timed out due to progressDeadlineSeconds of #{progress_seconds} seconds, "\
-        " reason: #{@progress['reason']}\n"\
-        "#{@latest_rs&.timeout_message}"
+      reason_msg = if @progress_condition.present?
+        "Timeout reason: #{@progress_condition['reason']}"
       else
-        @latest_rs&.timeout_message
+        "Timeout reason: hard deadline for #{type}"
       end
+      return reason_msg unless @latest_rs.present?
+      "#{reason_msg}\nLatest ReplicaSet: #{@latest_rs.name}\n\n#{@latest_rs.timeout_message}"
+    end
+
+    def pretty_timeout_type
+      @progress_deadline.present? ? "progress deadline: #{@progress_deadline}s" : super
     end
 
     def deploy_timed_out?
-      @progress ? @progress["status"] == 'False' : super
+      # Do not use the hard timeout if progress deadline is set
+      @progress_condition.present? ? deploy_failing_to_progress? : super
     end
 
     def exists?
@@ -72,6 +83,21 @@ module KubernetesDeploy
 
     private
 
+    def deploy_failing_to_progress?
+      return false unless @progress_condition.present?
+
+      if kubectl.server_version < Gem::Version.new("1.7.7")
+        # Deployments were being updated prematurely with incorrect progress information
+        # https://github.com/kubernetes/kubernetes/issues/49637
+        return false unless Time.now.utc - @deploy_started_at >= @progress_deadline.to_i
+      else
+        return false unless deploy_started?
+      end
+
+      @progress_condition["status"] == 'False' &&
+      Time.parse(@progress_condition["lastUpdateTime"]).to_i >= (@deploy_started_at - 5.seconds).to_i
+    end
+
     def find_latest_rs(deployment_data)
       label_string = deployment_data["spec"]["selector"]["matchLabels"].map { |k, v| "#{k}=#{v}" }.join(",")
       raw_json, _err, st = kubectl.run("get", "replicasets", "--output=json", "--selector=#{label_string}")
@@ -92,7 +118,7 @@ module KubernetesDeploy
         definition: latest_rs_data,
         logger: @logger,
         parent: "#{@name.capitalize} deployment",
-        deploy_started: @deploy_started
+        deploy_started_at: @deploy_started_at
       )
       rs.sync(latest_rs_data)
       rs

data/lib/kubernetes-deploy/kubernetes_resource/elasticsearch.rb

@@ -7,7 +7,7 @@ module KubernetesDeploy
     end
 
     def deploy_succeeded?
-      return false unless @deploy_started
+      return false unless deploy_started?
 
       unless @success_assumption_warning_shown
         @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.")

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -3,9 +3,11 @@ module KubernetesDeploy
   class Pod < KubernetesResource
     TIMEOUT = 10.minutes
 
-    def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started: nil)
+    FAILED_PHASE_NAME = "Failed"
+
+    def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started_at: nil)
       @parent = parent
-      @deploy_started = deploy_started
+      @deploy_started_at = deploy_started_at
       @containers = definition.fetch("spec", {}).fetch("containers", []).map { |c| Container.new(c) }
       unless @containers.present?
         logger.summary.add_paragraph("Rendered template content:\n#{definition.to_yaml}")
@@ -19,7 +21,7 @@ module KubernetesDeploy
       if pod_data.blank?
         raw_json, _err, st = kubectl.run("get", type, @name, "-a", "--output=json")
         pod_data = JSON.parse(raw_json) if st.success?
-        raise_predates_deploy_error if pod_data.present? && unmanaged? && !@deploy_started
+        raise_predates_deploy_error if pod_data.present? && unmanaged? && !deploy_started?
       end
 
       if pod_data.present?
@@ -46,8 +48,7 @@ module KubernetesDeploy
     end
 
     def deploy_failed?
-      return true if @phase == "Failed"
-      @containers.any?(&:doomed?)
+      failure_message.present?
     end
 
     def exists?
@@ -62,19 +63,23 @@ module KubernetesDeploy
     end
 
     def failure_message
-      doomed_containers = @containers.select(&:doomed?)
-      return unless doomed_containers.present?
-      container_messages = doomed_containers.map do |c|
-        red_name = ColorizedString.new(c.name).red
-        "> #{red_name}: #{c.doom_reason}"
+      if @phase == FAILED_PHASE_NAME
+        phase_problem = "Pod status: #{@status}. "
       end
 
-      intro = if unmanaged?
-        "The following containers encountered errors:"
-      else
-        "The following containers are in a state that is unlikely to be recoverable:"
+      doomed_containers = @containers.select(&:doomed?)
+      if doomed_containers.present?
+        container_problems = if unmanaged?
+          "The following containers encountered errors:\n"
+        else
+          "The following containers are in a state that is unlikely to be recoverable:\n"
+        end
+        doomed_containers.each do |c|
+          red_name = ColorizedString.new(c.name).red
+          container_problems += "> #{red_name}: #{c.doom_reason}\n"
+        end
       end
-      intro + "\n" + container_messages.join("\n") + "\n"
+      "#{phase_problem}#{container_problems}".presence
     end
 
     # Returns a hash in the following format:
@@ -89,7 +94,7 @@ module KubernetesDeploy
           "logs",
           @name,
           "--container=#{container.name}",
-          "--since-time=#{@deploy_started.to_datetime.rfc3339}",
+          "--since-time=#{@deploy_started_at.to_datetime.rfc3339}",
         ]
         cmd << "--tail=#{LOG_LINE_COUNT}" unless unmanaged?
         out, _err, _st = kubectl.run(*cmd)

data/lib/kubernetes-deploy/kubernetes_resource/pod_set_base.rb

@@ -23,7 +23,7 @@ module KubernetesDeploy
           "logs",
           id,
           "--container=#{container_name}",
-          "--since-time=#{@deploy_started.to_datetime.rfc3339}",
+          "--since-time=#{@deploy_started_at.to_datetime.rfc3339}",
           "--tail=#{LOG_LINE_COUNT}"
         )
         container_logs[container_name] = out.split("\n")
@@ -59,8 +59,8 @@ module KubernetesDeploy
           context: context,
           definition: pod_data,
           logger: @logger,
-          parent: "#{name.capitalize} #{self.class.name}",
-          deploy_started: @deploy_started
+          parent: "#{name.capitalize} #{type}",
+          deploy_started_at: @deploy_started_at
         )
         pod.sync(pod_data)
         relevant_pods << pod

data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb

@@ -5,10 +5,11 @@ module KubernetesDeploy
     TIMEOUT = 5.minutes
     attr_reader :desired_replicas, :pods
 
-    def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started: nil)
+    def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started_at: nil)
       @parent = parent
-      @deploy_started = deploy_started
+      @deploy_started_at = deploy_started_at
       @rollout_data = { "replicas" => 0 }
+      @desired_replicas = -1
       @pods = []
       super(namespace: namespace, context: context, definition: definition, logger: logger)
     end
@@ -22,8 +23,9 @@ module KubernetesDeploy
       if rs_data.present?
         @found = true
         @desired_replicas = rs_data["spec"]["replicas"].to_i
-        @rollout_data = { "replicas" => 0 }.merge(rs_data["status"]
-          .slice("replicas", "availableReplicas", "readyReplicas"))
+        @rollout_data = { "replicas" => 0 }.merge(
+          rs_data["status"].slice("replicas", "availableReplicas", "readyReplicas")
+        )
         @status = @rollout_data.map { |state_replicas, num| "#{num} #{state_replicas.chop.pluralize(num)}" }.join(", ")
         @pods = find_pods(rs_data)
       else # reset
@@ -31,6 +33,7 @@ module KubernetesDeploy
         @rollout_data = { "replicas" => 0 }
         @status = nil
         @pods = []
+        @desired_replicas = -1
       end
     end
 

data/lib/kubernetes-deploy/kubernetes_resource/statefulservice.rb

@@ -7,7 +7,7 @@ module KubernetesDeploy
     end
 
     def deploy_succeeded?
-      return false unless @deploy_started
+      return false unless deploy_started?
 
       unless @success_assumption_warning_shown
         @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.")

data/lib/kubernetes-deploy/kubernetes_resource/topic.rb

@@ -7,7 +7,7 @@ module KubernetesDeploy
     end
 
     def deploy_succeeded?
-      return false unless @deploy_started
+      return false unless deploy_started?
 
       unless @success_assumption_warning_shown
         @logger.warn("Don't know how to monitor resources of type #{type}. Assuming #{id} deployed successfully.")

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -24,11 +24,12 @@ module KubernetesDeploy
         delay_sync_until = Time.now.utc + delay_sync # don't pummel the API if the sync is fast
 
         KubernetesDeploy::Concurrency.split_across_threads(remainder, &:sync)
-        newly_finished_resources, remainder = remainder.partition(&:deploy_finished?)
+        new_successes, remainder = remainder.partition(&:deploy_succeeded?)
+        new_failures, remainder = remainder.partition(&:deploy_failed?)
+        new_timeouts, remainder = remainder.partition(&:deploy_timed_out?)
 
-        if newly_finished_resources.present?
-          watch_time = (Time.now.utc - @deploy_started_at).round(1)
-          report_what_just_happened(newly_finished_resources, watch_time)
+        if new_successes.present? || new_failures.present? || new_timeouts.present?
+          report_what_just_happened(new_successes, new_failures, new_timeouts)
           report_what_is_left(remainder, reminder: false)
           last_message_logged_at = Time.now.utc
         elsif due_for_reminder?(last_message_logged_at, reminder_interval)
@@ -41,19 +42,20 @@ module KubernetesDeploy
 
     private
 
-    def report_what_just_happened(resources, watch_time)
-      resources.each { |r| r.report_status_to_statsd(watch_time) }
-
-      new_successes, new_failures = resources.partition(&:deploy_succeeded?)
+    def report_what_just_happened(new_successes, new_failures, new_timeouts)
+      watch_time = (Time.now.utc - @deploy_started_at).round(1)
       new_failures.each do |resource|
-        if resource.deploy_failed?
-          @logger.error("#{resource.id} failed to #{@operation_name} after #{watch_time}s")
-        else
-          @logger.error("#{resource.id} rollout timed out")
-        end
+        resource.report_status_to_statsd(watch_time)
+        @logger.error("#{resource.id} failed to #{@operation_name} after #{watch_time}s")
+      end
+
+      new_timeouts.each do |resource|
+        resource.report_status_to_statsd(watch_time)
+        @logger.error("#{resource.id} rollout timed out after #{watch_time}s")
       end
 
       if new_successes.present?
+        new_successes.each { |r| r.report_status_to_statsd(watch_time) }
         success_string = ColorizedString.new("Successfully #{@operation_name}ed in #{watch_time}s:").green
         @logger.info("#{success_string} #{new_successes.map(&:id).join(', ')}")
       end

data/lib/kubernetes-deploy/restart_task.rb

@@ -80,7 +80,7 @@ module KubernetesDeploy
       kubeclient_resources.map do |d|
         definition = d.to_h.deep_stringify_keys
         r = Deployment.new(namespace: @namespace, context: @context, definition: definition, logger: @logger)
-        r.deploy_started = started # we don't care what happened to the resource before the restart cmd ran
+        r.deploy_started_at = started # we don't care what happened to the resource before the restart cmd ran
         r
       end
     end

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.13.0"
+  VERSION = "0.14.0"
 end

data/shipit.yml

@@ -0,0 +1,4 @@
+ci:
+  hide:
+    -  buildkite/kubernetes-deploy-gem-nightly
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-09 00:00:00.000000000 Z
+date: 2017-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -203,6 +203,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".buildkite/pipeline.nightly.yml"
 - ".buildkite/pipeline.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -224,6 +225,7 @@ files:
 - lib/kubernetes-deploy.rb
 - lib/kubernetes-deploy/concurrency.rb
 - lib/kubernetes-deploy/deferred_summary_logging.rb
+- lib/kubernetes-deploy/deploy_task.rb
 - lib/kubernetes-deploy/ejson_secret_provisioner.rb
 - lib/kubernetes-deploy/errors.rb
 - lib/kubernetes-deploy/formatted_logger.rb
@@ -255,7 +257,6 @@ files:
 - lib/kubernetes-deploy/kubernetes_resource/topic.rb
 - lib/kubernetes-deploy/resource_watcher.rb
 - lib/kubernetes-deploy/restart_task.rb
-- lib/kubernetes-deploy/runner.rb
 - lib/kubernetes-deploy/runner_task.rb
 - lib/kubernetes-deploy/statsd.rb
 - lib/kubernetes-deploy/version.rb
@@ -264,6 +265,7 @@ files:
 - screenshots/missing-secret-fail.png
 - screenshots/success.png
 - screenshots/test-output.png
+- shipit.yml
 homepage: https://github.com/Shopify/kubernetes-deploy
 licenses:
 - MIT