kubernetes-deploy 0.17.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a1b890ee465576a556e459c2d40850f7a8cab77
-  data.tar.gz: 20e835abd7b1f437e55217968cd91a84da8c1c32
+  metadata.gz: f0ae7839357ce28f77a544c170be1e439515205c
+  data.tar.gz: 945483435a69d88450f060d2a544cb046f93a17d
 SHA512:
-  metadata.gz: 67ccc21a2cacd5376b390b5f130613b3097785d0c474b72825c653f815faa2c4960010c1eadcad9aad7dc4ed71afaeab51cf4a0a23623b817590922a3a41c455
-  data.tar.gz: f039b9b67115242243082264c793a74722121407aca2d0f87a114cff4b4b1a33c8d2f8558513722bdb304876f6942a068b6d9a7e4f775ef2b311310977c7fb5a
+  metadata.gz: cc67d2dbc4ea9563f84bd11ae7c991144a0153223f21d07d14e6ad632396167d978c7a9a881386dc1a9a55756a5e06e4e9e60a1b1534b20b4705b4f38fb4d0a4
+  data.tar.gz: df05433e91bc432c9e7412c4806dbdfe4d5ba494763a0e4dfaf6ea6231ed1b1596c6e9dc4157e40f8dcf8857db828cbec9b098244964d147bcc84a2ce74c254e

data/.rubocop.yml

@@ -3,3 +3,9 @@ inherit_from:
 
 AllCops:
   TargetRubyVersion: 2.3
+
+Style/TrailingCommaInArrayLiteral:
+  Enabled: false
+
+Style/TrailingCommaInHashLiteral:
+  Enabled: false

data/CHANGELOG.md

@@ -1,7 +1,17 @@
+### 0.17.1
+*Features*
+- kubernetes-restart and kubernetes-deploy use exit code 70 when a
+deploy fails due to one or more resources failing to deploy in time.
+([#244](https://github.com/Shopify/kubernetes-deploy/pull/244))
+
+*Bug Fixes*
+- Handle deploying thousands of resources at a time, previously kubernetes-deploy would fail with
+ `Argument list too long - kubectl (Errno::E2BIG)`. ([#257](https://github.com/Shopify/kubernetes-deploy/pull/257))
+
 ### 0.17.0
 *Enhancements*
 
-- Add the `--cascade` when we force replace a resource. ([#250](https://github.com/Shopify/kubernetes-deploy/pull/250))
+- Add the `--cascade` flag when we force replace a resource. ([#250](https://github.com/Shopify/kubernetes-deploy/pull/250))
 
 ### 0.16.0
 **Important:** This release changes the officially supported Kubernetes versions to v1.7 through v1.9. Other versions may continue to work, but we are no longer running our test suite against them.

data/README.md

@@ -70,9 +70,9 @@ This repo also includes related tools for [running tasks](#kubernetes-run) and [
 
 * Ruby 2.3+
 * Your cluster must be running Kubernetes v1.7.0 or higher<sup>1</sup>
-* Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#templates))
-* You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory.**
-* Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.<sup>2</sup>
+* Each app must have a deploy directory containing its Kubernetes templates (see [Templates](#using-templates-and-variables))
+* You must remove the` kubectl.kubernetes.io/last-applied-configuration` annotation from any resources in the namespace that are not included in your deploy directory. This annotation is added automatically when you create resources with `kubectl apply`. `kubernetes-deploy` will prune any resources that have this annotation and are not in the deploy directory.<sup>2</sup>
+* Each app managed by `kubernetes-deploy` must have its own exclusive Kubernetes namespace.
 
 <sup>1</sup> We run integration tests against these Kubernetes versions. Kubernetes v1.6 was officially supported in gem versions < 0.16. Kubernetes v1.5 was officially supported in gem versions < 0.12.
 

data/exe/kubernetes-deploy

@@ -63,9 +63,14 @@ runner = KubernetesDeploy::DeployTask.new(
   logger: logger
 )
 
-success = runner.run(
-  verify_result: !skip_wait,
-  allow_protected_ns: allow_protected_ns,
-  prune: prune
-)
-exit 1 unless success
+begin
+  runner.run!(
+    verify_result: !skip_wait,
+    allow_protected_ns: allow_protected_ns,
+    prune: prune
+  )
+rescue KubernetesDeploy::DeploymentTimeoutError
+  exit 70
+rescue KubernetesDeploy::FatalDeploymentError
+  exit 1
+end

data/exe/kubernetes-restart

@@ -17,5 +17,10 @@ context = ARGV[1]
 logger = KubernetesDeploy::FormattedLogger.build(namespace, context)
 
 restart = KubernetesDeploy::RestartTask.new(namespace: namespace, context: context, logger: logger)
-success = restart.perform(raw_deployments)
-exit 1 unless success
+begin
+  restart.perform!(raw_deployments)
+rescue KubernetesDeploy::DeploymentTimeoutError
+  exit 70
+rescue KubernetesDeploy::FatalDeploymentError
+  exit 1
+end

data/lib/kubernetes-deploy/deferred_summary_logging.rb

@@ -34,12 +34,16 @@ module KubernetesDeploy
     end
 
     # Outputs the deferred summary information saved via @logger.summary.add_action and @logger.summary.add_paragraph
-    def print_summary(success)
-      if success
-        heading("Result: ", "SUCCESS", :green)
+    def print_summary(status)
+      status_string = status.to_s.humanize.upcase
+      if status == :success
+        heading("Result: ", status_string, :green)
         level = :info
+      elsif status == :timed_out
+        heading("Result: ", status_string, :yellow)
+        level = :fatal
       else
-        heading("Result: ", "FAILURE", :red)
+        heading("Result: ", status_string, :red)
         level = :fatal
       end
 

data/lib/kubernetes-deploy/deploy_task.rb

@@ -3,6 +3,7 @@ require 'open3'
 require 'yaml'
 require 'shellwords'
 require 'tempfile'
+require 'fileutils'
 require 'kubernetes-deploy/kubernetes_resource'
 %w(
   cloudsql
@@ -56,7 +57,6 @@ module KubernetesDeploy
       kube-system
       kube-public
     )
-
     # Things removed from default prune whitelist at https://github.com/kubernetes/kubernetes/blob/0dff56b4d88ec7551084bf89028dbeebf569620e/pkg/kubectl/cmd/apply.go#L411:
     # core/v1/Namespace -- not namespaced
     # core/v1/PersistentVolume -- not namespaced
@@ -105,7 +105,14 @@ module KubernetesDeploy
       )
     end
 
-    def run(verify_result: true, allow_protected_ns: false, prune: true)
+    def run(*args)
+      run!(*args)
+      true
+    rescue FatalDeploymentError
+      false
+    end
+
+    def run!(verify_result: true, allow_protected_ns: false, prune: true)
       start = Time.now.utc
       @logger.reset
 
@@ -148,7 +155,12 @@ module KubernetesDeploy
         start_normal_resource = Time.now.utc
         deploy_resources(resources, prune: prune, verify: true)
         ::StatsD.measure('normal_resources.duration', StatsD.duration(start_normal_resource), tags: statsd_tags)
-        success = resources.all?(&:deploy_succeeded?)
+        failed_resources = resources.reject(&:deploy_succeeded?)
+        success = failed_resources.empty?
+        if !success && failed_resources.all?(&:deploy_timed_out?)
+          raise DeploymentTimeoutError
+        end
+        raise FatalDeploymentError unless success
       else
         deploy_resources(resources, prune: prune, verify: false)
         @logger.summary.add_action("deployed #{resources.length} #{'resource'.pluralize(resources.length)}")
@@ -157,16 +169,18 @@ module KubernetesDeploy
           This means the desired changes were communicated to Kubernetes, but the deploy did not make sure they actually succeeded.
         MSG
         @logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
-        success = true
       end
+      ::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:success")
+      @logger.print_summary(:success)
+    rescue DeploymentTimeoutError
+      @logger.print_summary(:timed_out)
+      ::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:timeout")
+      raise
     rescue FatalDeploymentError => error
-      @logger.summary.add_action(error.message)
-      success = false
-    ensure
-      @logger.print_summary(success)
-      status = success ? "success" : "failed"
-      ::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:#{status}")
-      success
+      @logger.summary.add_action(error.message) if error.message != error.class.to_s
+      @logger.print_summary(:failure)
+      ::StatsD.measure('all_resources.duration', StatsD.duration(start), tags: statsd_tags << "status:failed")
+      raise
     end
 
     private
@@ -366,29 +380,33 @@ module KubernetesDeploy
 
     def apply_all(resources, prune)
       return unless resources.present?
-
       command = ["apply"]
-      resources.each do |r|
-        @logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
-        command.push("-f", r.file_path)
-        r.deploy_started_at = Time.now.utc
-      end
 
-      if prune
-        command.push("--prune", "--all")
-        prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
-      end
+      Dir.mktmpdir do |tmp_dir|
+        resources.each do |r|
+          @logger.info("- #{r.id} (#{r.pretty_timeout_type})") if resources.length > 1
+          FileUtils.symlink(r.file_path, tmp_dir)
+          r.deploy_started_at = Time.now.utc
+        end
+        command.push("-f", tmp_dir)
 
-      out, err, st = kubectl.run(*command, log_failure: false)
-      if st.success?
-        log_pruning(out) if prune
-      else
-        file_paths = find_bad_files_from_kubectl_output(err)
-        warn_msg = "WARNING: Any resources not mentioned in the error below were likely created/updated. " \
-          "You may wish to roll back this deploy."
-        @logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
-        record_invalid_template(err, file_paths: file_paths)
-        raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
+        if prune
+          command.push("--prune", "--all")
+          prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
+        end
+
+        out, err, st = kubectl.run(*command, log_failure: false)
+
+        if st.success?
+          log_pruning(out) if prune
+        else
+          file_paths = find_bad_files_from_kubectl_output(err)
+          warn_msg = "WARNING: Any resources not mentioned in the error below were likely created/updated. " \
+            "You may wish to roll back this deploy."
+          @logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
+          record_invalid_template(err, file_paths: file_paths)
+          raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
+        end
       end
     end
 

data/lib/kubernetes-deploy/errors.rb

@@ -8,6 +8,9 @@ module KubernetesDeploy
       super("Namespace `#{name}` not found in context `#{context}`")
     end
   end
+
+  class DeploymentTimeoutError < FatalDeploymentError; end
+
   module Errors
     extend self
     def server_version_warning(server_version)

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -81,7 +81,18 @@ module KubernetesDeploy
       end
 
       if fail_count > 0
-        @logger.summary.add_action("failed to #{@operation_name} #{fail_count} #{'resource'.pluralize(fail_count)}")
+        timeouts, failures = failed_resources.partition(&:deploy_timed_out?)
+        if timeouts.present?
+          @logger.summary.add_action(
+            "timed out waiting for #{timeouts.length} #{'resource'.pluralize(timeouts.length)} to #{@operation_name}"
+          )
+        end
+
+        if failures.present?
+          @logger.summary.add_action(
+            "failed to #{@operation_name} #{failures.length} #{'resource'.pluralize(failures.length)}"
+          )
+        end
         KubernetesDeploy::Concurrency.split_across_threads(failed_resources, &:sync_debug_info)
         failed_resources.each { |r| @logger.summary.add_paragraph(r.debug_message) }
       end

data/lib/kubernetes-deploy/restart_task.rb

@@ -26,7 +26,14 @@ module KubernetesDeploy
       @logger = logger
     end
 
-    def perform(deployments_names = nil)
+    def perform(*args)
+      perform!(*args)
+      true
+    rescue FatalDeploymentError
+      false
+    end
+
+    def perform!(deployments_names = nil)
       start = Time.now.utc
       @logger.reset
 
@@ -42,19 +49,31 @@ module KubernetesDeploy
       @logger.phase_heading("Waiting for rollout")
       resources = build_watchables(deployments, start)
       ResourceWatcher.new(resources, logger: @logger, operation_name: "restart").run
-      success = resources.all?(&:deploy_succeeded?)
+      failed_resources = resources.reject(&:deploy_succeeded?)
+      success = failed_resources.empty?
+      if !success && failed_resources.all?(&:deploy_timed_out?)
+        raise DeploymentTimeoutError
+      end
+      raise FatalDeploymentError unless success
+      ::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('success', deployments))
+      @logger.print_summary(:success)
+    rescue DeploymentTimeoutError
+      ::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('timeout', deployments))
+      @logger.print_summary(:timed_out)
+      raise
     rescue FatalDeploymentError => error
-      @logger.summary.add_action(error.message)
-      success = false
-    ensure
-      @logger.print_summary(success)
-      status = success ? "success" : "failed"
-      tags = %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
-      ::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags)
+      ::StatsD.measure('restart.duration', StatsD.duration(start), tags: tags('failure', deployments))
+      @logger.summary.add_action(error.message) if error.message != error.class.to_s
+      @logger.print_summary(:failure)
+      raise
     end
 
     private
 
+    def tags(status, deployments)
+      %W(namespace:#{@namespace} context:#{@context} status:#{status} deployments:#{deployments.to_a.length}})
+    end
+
     def identify_target_deployments(deployment_names)
       if deployment_names.nil?
         @logger.info("Configured to restart all deployments with the `#{ANNOTATION}` annotation")

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.17.0"
+  VERSION = "0.18.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.18.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-13 00:00:00.000000000 Z
+date: 2018-04-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport