kubernetes-deploy 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99f56eb5da1d7f17885cff3d19787b45a1484504
-  data.tar.gz: e2919b899b313e2c4be0af2b7731816f9c8336de
+  metadata.gz: a581b73a9b59ce7dbc35a528acf269ba9ee12523
+  data.tar.gz: 4d3b6268317bf30b3011d1eb3e9f8bcc9c731c4f
 SHA512:
-  metadata.gz: 103546e46a2e476e1cc1ae339fcce8362fd199d144d890d1b895ed46848181ba822dcb713eb672581f625b25e3992c4ea9439e59888c4b3f0b74b44d156bba50
-  data.tar.gz: 7aeb474292e52f2b3e9ead4f038732804949c253da2119993c89d64e1357d7920c2031bce184821cb711c091ed54f8b6edf181c47d72b9242c3d2e0c972d7355
+  metadata.gz: 15197ca3f8d90b28927dc8a5ab365bb7f455434c49af708db82b2f8f5dd7430420a8e4819edbc1d4579cd2291df0a002f11924be3a865631a11e5a1fa3f3b344
+  data.tar.gz: bae4f408b4bfad19e61754991f24bf34d48dca2929f1d33dbd592b2fcc009b7135adc3dc92201c1e8d7633d80520a798e03aedec45778d59d81aa0a872ab7426

data/.buildkite/pipeline.nightly.yml

@@ -1,10 +1,11 @@
-  - name: 'Run Test Suite (:kubernetes: 1.10-latest)'
-    command: bin/ci
-    agents:
-      queue: minikube-ci
-    env:
-      LOGGING_LEVEL: 4
-      KUBERNETES_VERSION: v1.10-latest
+# Disableed until minikube can run 1.10
+#  - name: 'Run Test Suite (:kubernetes: 1.10-latest)'
+#    command: bin/ci
+#    agents:
+#      queue: minikube-ci
+#    env:
+#      LOGGING_LEVEL: 4
+#      KUBERNETES_VERSION: v1.10-latest
   - name: 'Run Test Suite (:kubernetes: 1.9-latest)'
     command: bin/ci
     agents:

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+### Master
+
+*Features*
+
+*Bug Fixes*
+
+*Enhancements*
+
+### 0.20.0
+
+*Features*
+- Automatically add all Kubernetes namespace labels to StatsD tags ([#278](https://github.com/Shopify/kubernetes-deploy/pull/278))
+
+*Bug Fixes*
+- Prevent calling sleep with a negative value ([#273](https://github.com/Shopify/kubernetes-deploy/pull/273))
+- Prevent no-op redeploys of bad code from hanging forever ([#262](https://github.com/Shopify/kubernetes-deploy/pull/262))
+
+*Enhancements*
+- Improve output for rendering errors ([#253](https://github.com/Shopify/kubernetes-deploy/pull/253))
+
 ### 0.19.0
 *Features*
 - Added `--max-watch-seconds=seconds` to kubernetes-restart and kubernetes-deploy. When set

data/dev.yml

@@ -8,7 +8,7 @@ up:
   - custom:
       name: Minikube Cluster
       met?: test $(minikube status | grep Running | wc -l) -eq 2 && $(minikube status | grep -q 'Correctly Configured')
-      meet: minikube start --vm-driver=xhyve --kubernetes-version=v1.7.5
+      meet: minikube start --kubernetes-version=v1.9.4 --vm-driver=hyperkit
       down: minikube stop
 commands:
   reset-minikube: minikube delete && rm -rf ~/.minikube

data/kubernetes-deploy.gemspec

@@ -36,5 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "minitest-stub-const", "~> 0.6"
   spec.add_development_dependency "webmock", "~> 3.0"
-  spec.add_development_dependency "mocha", "~> 1.1"
+  spec.add_development_dependency "mocha", "~> 1.5"
 end

data/lib/kubernetes-deploy/deploy_task.rb

@@ -92,6 +92,7 @@ module KubernetesDeploy
     def initialize(namespace:, context:, current_sha:, template_dir:, logger:, kubectl_instance: nil, bindings: {},
       max_watch_seconds: nil)
       @namespace = namespace
+      @namespace_tags = []
       @context = context
       @current_sha = current_sha
       @template_dir = File.expand_path(template_dir)
@@ -122,6 +123,7 @@ module KubernetesDeploy
       validate_configuration(allow_protected_ns: allow_protected_ns, prune: prune)
       confirm_context_exists
       confirm_namespace_exists
+      @namespace_tags |= tags_from_namespace_labels
       resources = discover_resources
       validate_definitions(resources)
 
@@ -187,15 +189,6 @@ module KubernetesDeploy
 
     private
 
-    # Inspect the file referenced in the kubectl stderr
-    # to make it easier for developer to understand what's going on
-    def find_bad_files_from_kubectl_output(stderr)
-      # stderr often contains one or more lines like the following, from which we can extract the file path(s):
-      # Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
-      matches = stderr.scan(%r{"(/\S+\.ya?ml\S*)"})
-      matches&.flatten
-    end
-
     def deploy_has_priority_resources?(resources)
       resources.any? { |r| PREDEPLOY_SEQUENCE.include?(r.type) }
     end
@@ -225,7 +218,8 @@ module KubernetesDeploy
       return unless failed_resources.present?
 
       failed_resources.each do |r|
-        record_invalid_template(r.validation_error_msg, file_paths: [r.file_path])
+        content = File.read(r.file_path) if File.file?(r.file_path)
+        record_invalid_template(err: r.validation_error_msg, filename: File.basename(r.file_path), content: content)
       end
       raise FatalDeploymentError, "Template validation failed"
     end
@@ -238,7 +232,8 @@ module KubernetesDeploy
         next unless filename.end_with?(".yml.erb", ".yml", ".yaml", ".yaml.erb")
 
         split_templates(filename) do |r_def|
-          r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger, definition: r_def)
+          r = KubernetesResource.build(namespace: @namespace, context: @context, logger: @logger,
+                                       definition: r_def, statsd_tags: @namespace_tags)
           resources << r
           @logger.info "  - #{r.id}"
         end
@@ -250,34 +245,25 @@ module KubernetesDeploy
       file_content = File.read(File.join(@template_dir, filename))
       rendered_content = @renderer.render_template(filename, file_content)
       YAML.load_stream(rendered_content) do |doc|
-        yield doc unless doc.blank?
+        next if doc.blank?
+        unless doc.is_a?(Hash)
+          raise InvalidTemplateError.new("Template is not a valid Kubernetes manifest",
+            filename: filename, content: doc)
+        end
+        yield doc
       end
+    rescue InvalidTemplateError => e
+      record_invalid_template(err: e.message, filename: e.filename, content: e.content)
+      raise FatalDeploymentError, "Failed to render and parse template"
     rescue Psych::SyntaxError => e
-      debug_msg = <<~INFO
-        Error message: #{e}
-
-        Template content:
-        ---
-      INFO
-      debug_msg += rendered_content
-      @logger.summary.add_paragraph(debug_msg)
-      raise FatalDeploymentError, "Template '#{filename}' cannot be parsed"
+      record_invalid_template(err: e.message, filename: filename, content: rendered_content)
+      raise FatalDeploymentError, "Failed to render and parse template"
     end
 
-    def record_invalid_template(err, file_paths:, original_filenames: nil)
-      template_names = Array(original_filenames)
-      file_content = Array(file_paths).each_with_object([]) do |file_path, contents|
-        next unless File.file?(file_path)
-        contents << File.read(file_path)
-        template_names << File.basename(file_path) unless original_filenames
-      end.join("\n")
-      template_list = template_names.compact.join(", ").presence || "See error message"
-
-      debug_msg = ColorizedString.new("Invalid #{'template'.pluralize(template_names.length)}: #{template_list}\n").red
-      debug_msg += "> Error from kubectl:\n#{indent_four(err)}"
-      if file_content.present?
-        debug_msg += "\n> Rendered template content:\n#{indent_four(file_content)}"
-      end
+    def record_invalid_template(err:, filename:, content:)
+      debug_msg = ColorizedString.new("Invalid template: #{filename}\n").red
+      debug_msg += "> Error message:\n#{indent_four(err)}"
+      debug_msg += "\n> Template content:\n#{indent_four(content)}"
       @logger.summary.add_paragraph(debug_msg)
     end
 
@@ -406,11 +392,7 @@ module KubernetesDeploy
         if st.success?
           log_pruning(out) if prune
         else
-          file_paths = find_bad_files_from_kubectl_output(err)
-          warn_msg = "WARNING: Any resources not mentioned in the error below were likely created/updated. " \
-            "You may wish to roll back this deploy."
-          @logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
-          record_invalid_template(err, file_paths: file_paths)
+          record_apply_failure(err)
           raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
         end
       end
@@ -424,6 +406,41 @@ module KubernetesDeploy
       @logger.summary.add_action("pruned #{pruned.length} #{'resource'.pluralize(pruned.length)}")
     end
 
+    def record_apply_failure(err)
+      warn_msg = "WARNING: Any resources not mentioned in the error(s) below were likely created/updated. " \
+        "You may wish to roll back this deploy."
+      @logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
+
+      unidentified_errors = []
+      err.each_line do |line|
+        bad_files = find_bad_files_from_kubectl_output(line)
+        if bad_files.present?
+          bad_files.each { |f| record_invalid_template(err: f[:err], filename: f[:filename], content: f[:content]) }
+        else
+          unidentified_errors << line
+        end
+      end
+
+      if unidentified_errors.present?
+        msg = "#{ColorizedString.new('Unidentified error(s):').red}\n#{indent_four(unidentified_errors.join)}"
+        @logger.summary.add_paragraph(msg)
+      end
+    end
+
+    # Inspect the file referenced in the kubectl stderr
+    # to make it easier for developer to understand what's going on
+    def find_bad_files_from_kubectl_output(line)
+      # stderr often contains one or more lines like the following, from which we can extract the file path(s):
+      # Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
+
+      line.scan(%r{"(/\S+\.ya?ml\S*)"}).each_with_object([]) do |matches, bad_files|
+        matches.each do |path|
+          content = File.read(path) if File.file?(path)
+          bad_files << { filename: File.basename(path), err: line, content: content }
+        end
+      end
+    end
+
     def confirm_context_exists
       out, err, st = kubectl.run("config", "get-contexts", "-o", "name",
         use_namespace: false, use_context: false, log_failure: false)
@@ -462,12 +479,24 @@ module KubernetesDeploy
       @logger.info("Namespace #{@namespace} found")
     end
 
+    def tags_from_namespace_labels
+      namespace_info = nil
+      with_retries(2) do
+        namespace_info, _, st = kubectl.run("get", "namespace", @namespace, "-o", "json", use_namespace: false,
+          log_failure: true)
+        st.success?
+      end
+      return [] if namespace_info.blank?
+      namespace_labels = JSON.parse(namespace_info, symbolize_names: true).fetch(:metadata, {}).fetch(:labels, {})
+      namespace_labels.map { |key, value| "#{key}:#{value}" }
+    end
+
     def kubectl
       @kubectl ||= Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: true)
     end
 
     def statsd_tags
-      %W(namespace:#{@namespace} sha:#{@current_sha} context:#{@context})
+      %W(namespace:#{@namespace} sha:#{@current_sha} context:#{@context}) | @namespace_tags
     end
 
     def with_retries(limit)

data/lib/kubernetes-deploy/errors.rb

@@ -3,6 +3,15 @@ module KubernetesDeploy
   class FatalDeploymentError < StandardError; end
   class KubectlError < StandardError; end
 
+  class InvalidTemplateError < FatalDeploymentError
+    attr_reader :filename, :content
+    def initialize(err, filename:, content: nil)
+      @filename = filename
+      @content = content
+      super(err)
+    end
+  end
+
   class NamespaceNotFoundError < FatalDeploymentError
     def initialize(name, context)
       super("Namespace `#{name}` not found in context `#{context}`")

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -28,8 +28,9 @@ module KubernetesDeploy
     TIMEOUT_OVERRIDE_ANNOTATION = "kubernetes-deploy.shopify.io/timeout-override"
 
     class << self
-      def build(namespace:, context:, definition:, logger:)
-        opts = { namespace: namespace, context: context, definition: definition, logger: logger }
+      def build(namespace:, context:, definition:, logger:, statsd_tags:)
+        opts = { namespace: namespace, context: context, definition: definition, logger: logger,
+                 statsd_tags: statsd_tags }
         if KubernetesDeploy.const_defined?(definition["kind"])
           klass = KubernetesDeploy.const_get(definition["kind"])
           klass.new(**opts)
@@ -65,7 +66,7 @@ module KubernetesDeploy
       "timeout: #{timeout}s"
     end
 
-    def initialize(namespace:, context:, definition:, logger:)
+    def initialize(namespace:, context:, definition:, logger:, statsd_tags: [])
       # subclasses must also set these if they define their own initializer
       @name = definition.dig("metadata", "name")
       unless @name.present?
@@ -73,6 +74,7 @@ module KubernetesDeploy
         raise FatalDeploymentError, "Template is missing required field metadata.name"
       end
 
+      @optional_statsd_tags = statsd_tags
       @namespace = namespace
       @context = context
       @logger = logger
@@ -356,7 +358,9 @@ module KubernetesDeploy
       else
         "unknown"
       end
-      %W(context:#{context} namespace:#{namespace} resource:#{id} type:#{type} sha:#{ENV['REVISION']} status:#{status})
+      tags = %W(context:#{context} namespace:#{namespace} resource:#{id}
+                type:#{type} sha:#{ENV['REVISION']} status:#{status})
+      tags | @optional_statsd_tags
     end
   end
 end

data/lib/kubernetes-deploy/kubernetes_resource/deployment.rb

@@ -96,6 +96,16 @@ module KubernetesDeploy
 
     private
 
+    def current_generation
+      return -2 unless exists? # different default than observed
+      @instance_data.dig('metadata', 'generation')
+    end
+
+    def observed_generation
+      return -1 unless exists? # different default than current
+      @instance_data.dig('status', 'observedGeneration')
+    end
+
     def desired_replicas
       return -1 unless exists?
       @instance_data["spec"]["replicas"].to_i
@@ -133,12 +143,13 @@ module KubernetesDeploy
         # Deployments were being updated prematurely with incorrect progress information
         # https://github.com/kubernetes/kubernetes/issues/49637
         return false unless Time.now.utc - @deploy_started_at >= progress_deadline.to_i
-      else
-        return false unless deploy_started?
       end
 
-      progress_condition["status"] == 'False' &&
-      Time.parse(progress_condition["lastUpdateTime"]).to_i >= (@deploy_started_at - 5.seconds).to_i
+      # This assumes that when the controller bumps the observed generation, it also updates/clears all the status
+      # conditions. Specifically, it assumes the progress condition is immediately set to True if a rollout is starting.
+      deploy_started? &&
+      current_generation == observed_generation &&
+      progress_condition["status"] == 'False'
     end
 
     def find_latest_rs(mediator)

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -5,7 +5,8 @@ module KubernetesDeploy
 
     FAILED_PHASE_NAME = "Failed"
 
-    def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started_at: nil)
+    def initialize(namespace:, context:, definition:, logger:,
+      statsd_tags: nil, parent: nil, deploy_started_at: nil)
       @parent = parent
       @deploy_started_at = deploy_started_at
       @containers = definition.fetch("spec", {}).fetch("containers", []).map { |c| Container.new(c) }
@@ -14,7 +15,8 @@ module KubernetesDeploy
         raise FatalDeploymentError, "Template is missing required field spec.containers"
       end
       @containers += definition["spec"].fetch("initContainers", []).map { |c| Container.new(c, init_container: true) }
-      super(namespace: namespace, context: context, definition: definition, logger: logger)
+      super(namespace: namespace, context: context, definition: definition,
+            logger: logger, statsd_tags: statsd_tags)
     end
 
     def sync(mediator)

data/lib/kubernetes-deploy/kubernetes_resource/replica_set.rb

@@ -5,11 +5,13 @@ module KubernetesDeploy
     TIMEOUT = 5.minutes
     attr_reader :pods
 
-    def initialize(namespace:, context:, definition:, logger:, parent: nil, deploy_started_at: nil)
+    def initialize(namespace:, context:, definition:, logger:, statsd_tags: nil,
+      parent: nil, deploy_started_at: nil)
       @parent = parent
       @deploy_started_at = deploy_started_at
       @pods = []
-      super(namespace: namespace, context: context, definition: definition, logger: logger)
+      super(namespace: namespace, context: context, definition: definition,
+            logger: logger, statsd_tags: statsd_tags)
     end
 
     SYNC_DEPENDENCIES = %w(Pod)

data/lib/kubernetes-deploy/renderer.rb

@@ -7,14 +7,14 @@ require 'json'
 
 module KubernetesDeploy
   class Renderer
-    class InvalidPartialError < FatalDeploymentError
-      attr_reader :parents
-      def initialize(msg, parents = [])
+    class InvalidPartialError < InvalidTemplateError
+      attr_accessor :parents, :content, :filename
+      def initialize(msg, parents: [], content: nil, filename:)
         @parents = parents
-        super(msg)
+        super(msg, content: content, filename: filename)
       end
     end
-    class PartialNotFound < InvalidPartialError; end
+    class PartialNotFound < InvalidTemplateError; end
 
     def initialize(current_sha:, template_dir:, logger:, bindings: {})
       @current_sha = current_sha
@@ -35,11 +35,11 @@ module KubernetesDeploy
 
       ERB.new(raw_template, nil, '-').result(erb_binding)
     rescue InvalidPartialError => err
-      all_parents = err.parents.dup.unshift(filename)
-      raise FatalDeploymentError, "#{err.message} (included from: #{all_parents.join(' -> ')})"
+      err.parents = err.parents.dup.unshift(filename)
+      err.filename = "#{err.filename} (partial included from: #{err.parents.join(' -> ')})"
+      raise err
     rescue StandardError => err
-      report_template_invalid(err.message, raw_template)
-      raise FatalDeploymentError, "Template '#{filename}' cannot be rendered"
+      raise InvalidTemplateError.new(err.message, filename: filename, content: raw_template)
     end
 
     def render_partial(partial, locals)
@@ -60,12 +60,14 @@ module KubernetesDeploy
       # Note that JSON is a subset of YAML.
       JSON.generate(docs.children.first.to_ruby)
     rescue PartialNotFound => err
-      raise InvalidPartialError, err.message
+      # get the filename from the first parent, not the missing partial itself
+      raise err if err.filename == partial
+      raise InvalidPartialError.new(err.message, filename: partial, content: expanded_template || template)
     rescue InvalidPartialError => err
-      raise InvalidPartialError.new(err.message, err.parents.dup.unshift(File.basename(partial_path)))
+      err.parents = err.parents.dup.unshift(File.basename(partial_path))
+      raise err
     rescue StandardError => err
-      report_template_invalid(err.message, expanded_template)
-      raise InvalidPartialError, "Template '#{partial_path}' cannot be rendered"
+      raise InvalidPartialError.new(err.message, filename: partial_path, content: expanded_template || template)
     end
 
     private
@@ -91,12 +93,8 @@ module KubernetesDeploy
           return partial_path if File.exist?(partial_path)
         end
       end
-      raise PartialNotFound, "Could not find partial '#{name}' in any of #{@partials_dirs.join(':')}"
-    end
-
-    def report_template_invalid(message, content)
-      @logger.summary.add_paragraph("Error from renderer:\n  #{message.tr("\n", ' ')}")
-      @logger.summary.add_paragraph("Rendered template content:\n#{content}")
+      raise PartialNotFound.new("Could not find partial '#{name}' in any of #{@partials_dirs.join(':')}",
+        filename: name)
     end
 
     class TemplateContext

data/lib/kubernetes-deploy/resource_watcher.rb

@@ -24,8 +24,8 @@ module KubernetesDeploy
         if @timeout && (Time.now.utc - monitoring_started > @timeout)
           report_and_give_up(remainder)
         end
-        if Time.now.utc < delay_sync_until
-          sleep(delay_sync_until - Time.now.utc)
+        if (sleep_duration = delay_sync_until - Time.now.utc) > 0
+          sleep(sleep_duration)
         end
         delay_sync_until = Time.now.utc + delay_sync # don't pummel the API if the sync is fast
 

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.19.0"
+  VERSION = "0.20.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-09 00:00:00.000000000 Z
+date: 2018-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -185,14 +185,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.5'
 description: Kubernetes deploy scripts
 email:
 - ops-accounts+shipit@shopify.com