kubernetes-deploy 0.26.3 → 0.26.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dbfc9ddacc1877541f3c3d580be4e4f8dd2e4bbb59f6bbe52232ccc77985df1e
-  data.tar.gz: 1b15550b1ba3eada31790eb1b5521be3f6a9837108e379f97eb9791548b1478a
+  metadata.gz: a537ded5641b9cb4b9813594e6da9d7ea06876721b26ce4785e48f0b933536cf
+  data.tar.gz: 8ee185bc26198298d5ed8c18aba4736a67bc46bd2526a72d71b74c988ac9e657
 SHA512:
-  metadata.gz: 3f7fe96e6737b17e2d979ca6cbf6c21b8f2524a4029d0a1756c859029a003688ad0b51e6d9f55ac34190ca635a9b071a657b3654d3c7f2793627505e0cd5942c
-  data.tar.gz: 438181bf2d2e491c0ffb5565a96f98b4d2d9352b6a88958fae80cfaedc1590a11cd0a768f1e35878a59f13ec5e57b18f3d8ce89459ba16be014d41c706d5e5b4
+  metadata.gz: 33f35406225136f90fa99219f5545d9052e2d323055f87d0d42c81640549cda6f18c0748d661ae4ba4c4ab7a1bcb0e7c717e6c8081ded664849befd2bf571d3c
+  data.tar.gz: 3d3e78256ad83ea456f9a92106c81b03c0212fc3d0940cdfbb990a45214f7250af1110c4f3231dde8b828ab4113b68c3ce5dfc28d675a001d19eea686b2bb9ca

data/.rubocop.yml

@@ -8,3 +8,6 @@ Naming/FileName:
   Enabled: true
   Exclude:
     - lib/kubernetes-deploy.rb
+
+Sorbet/ConstantsFromStrings:
+  Enabled: false

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 ## next
 
+## 0.26.4
+
+*Bug fixes*
+- Adds several additional safeguards against the content of Secret resources being logged. [#474](https://github.com/Shopify/kubernetes-deploy/pull/474)
+
+*Enhancements*
+- Improves scalability by removing a check that caused recoverable registry problems to fail deploys. [#477](https://github.com/Shopify/kubernetes-deploy/pull/477)
+
+*Other*
+- Relaxes our dependency on the OJ gem. [#471](https://github.com/Shopify/kubernetes-deploy/pull/471)
+
 ## 0.26.3
 
 *Bug fixes*

data/CONTRIBUTING.md

@@ -113,8 +113,9 @@ If you work for Shopify, just run `dev up`, but otherwise:
 
 1. [Install kubectl version 1.10.0 or higher](https://kubernetes.io/docs/user-guide/prereqs/) and make sure it is in your path
 2. [Install minikube](https://kubernetes.io/docs/getting-started-guides/minikube/#installation) (required to run the test suite)
-3. Check out the repo
-4. Run `bin/setup` to install dependencies
+3. [Install any required minikube drivers](https://github.com/kubernetes/minikube/blob/master/docs/drivers.md) (on OS X, you may need the [hyperkit driver](https://github.com/kubernetes/minikube/blob/master/docs/drivers.md#hyperkit-driver)
+4. Check out the repo
+5. Run `bin/setup` to install dependencies
 
 To install this gem onto your local machine, run `bundle exec rake install`.
 
@@ -144,14 +145,11 @@ To see the full-color output of a specific integration test, you can use `PRINT_
 
 ## Releasing a new version (Shopify employees)
 
-1. Make sure all merged PRs are reflected in the changelog before creating the commit for the new version.
-2. Update the version number in `version.rb` and commit that change with message "Version x.y.z". Don't push yet or you'll confuse Shipit.
-3. Tag the version with `git tag vx.y.z -a -m "Version x.y.z"`
-4. Push both your bump commit and its tag simultaneously with `git push origin master --follow-tags` (note that you can set `git config --global push.followTags true` to turn this flag on by default)
-5. Use the [Shipit Stack](https://shipit.shopify.io/shopify/kubernetes-deploy/rubygems) to build the `.gem` file and upload to [rubygems.org](https://rubygems.org/gems/kubernetes-deploy).
-
-If you push your commit and the tag separately, Shipit usually fails with `You need to create the v0.7.9 tag first.`. To make it find your tag, go to `Settings` > `Resynchronize this stack` > `Clear git cache`.
-
+1. On a new branch, create a new heading in CHANGELOG.md for your version and move the entries from "Next" under it. Leave the "Next" heading in the file (this helps with the diff for rebases after the release).
+1. Make sure CHANGELOG.md includes all user-facing changes since the last release. Things like test changes or refactors do not need to be included.
+1. Update the version number in `version.rb`.
+1. Commit your changes with message "Version x.y.z" and open a PR.
+1. After merging your PR, deploy via [Shipit](https://shipit.shopify.io/shopify/kubernetes-deploy/rubygems). Shipit will automatically tag the release and upload the gem to [rubygems.org](https://rubygems.org/gems/kubernetes-deploy).
 
 ## CI (External contributors)
 

data/README.md

@@ -114,6 +114,8 @@ Refer to `kubernetes-deploy --help` for the authoritative set of options.
 resource to deploy.
 - `--selector`: Instructs kubernetes-deploy to only prune resources which match the specified label selector, such as `environment=staging`. If you use this option, all resource templates must specify matching labels. See [Sharing a namespace](#sharing-a-namespace) below.
 
+> **NOTICE**: Deploy Secret resources at your own risk. Although we will fix any reported leak vectors with urgency, we cannot guarantee that sensitive information will never be logged.
+
 ### Sharing a namespace
 
 By default, kubernetes-deploy will prune any resources in the target namespace which have the `kubectl.kubernetes.io/last-applied-configuration` annotation and are not a result of the current deployment process, on the assumption that there is a one-to-one relationship between application deployment and namespace, and that a deployment provisions all relevant resources in the namespace.

data/dev.yml

@@ -5,6 +5,10 @@ up:
   - bundler
   - homebrew:
     - Caskroom/cask/minikube
+  - custom:
+      name: Install the minikube fork of driver-hyperkit
+      met?: command -v docker-machine-driver-hyperkit
+      meet: curl -LO https://storage.googleapis.com/minikube/releases/latest/docker-machine-driver-hyperkit && sudo install -o root -g wheel -m 4755 docker-machine-driver-hyperkit /usr/local/bin/ && rm ./docker-machine-driver-hyperkit
   - custom:
       name: Minikube Cluster
       met?: test $(minikube status | grep Running | wc -l) -ge 2 && $(minikube status | grep -q 'Correctly Configured')

data/exe/kubernetes-render

@@ -12,7 +12,7 @@ ARGV.options do |opts|
   parser = KubernetesDeploy::BindingsParser.new
   opts.on("--bindings=BINDINGS", "Expose additional variables to ERB templates " \
     "(format: k1=v1,k2=v2, JSON string or file (JSON or YAML) path prefixed by '@')") { |b| parser.add(b) }
-  opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT)." ) do |d|
+  opts.on("--template-dir=DIR", "Set the template dir (default: config/deploy/$ENVIRONMENT).") do |d|
     template_dir = d
   end
   opts.parse!

data/kubernetes-deploy.gemspec

@@ -29,7 +29,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency("ejson", "~> 1.0")
   spec.add_dependency("colorize", "~> 0.8")
   spec.add_dependency("statsd-instrument", '~> 2.3', '>= 2.3.2')
-  spec.add_dependency("oj", "~> 3.7")
+  spec.add_dependency("oj", "~> 3.0")
   spec.add_dependency("concurrent-ruby", "~> 1.1")
   spec.add_dependency("jsonpath", "~> 0.9.6")
 

data/lib/kubernetes-deploy/deferred_summary_logging.rb

@@ -47,7 +47,7 @@ module KubernetesDeploy
         level = :fatal
       end
 
-      if actions_sentence = summary.actions_sentence.presence
+      if (actions_sentence = summary.actions_sentence.presence)
         public_send(level, actions_sentence)
         blank_line(level)
       end

data/lib/kubernetes-deploy/deploy_task.rb

@@ -252,7 +252,7 @@ module KubernetesDeploy
       return unless failed_resources.present?
 
       failed_resources.each do |r|
-        content = File.read(r.file_path) if File.file?(r.file_path)
+        content = File.read(r.file_path) if File.file?(r.file_path) && !r.sensitive_template_content?
         record_invalid_template(err: r.validation_error_msg, filename: File.basename(r.file_path), content: content)
       end
       raise FatalDeploymentError, "Template validation failed"
@@ -319,7 +319,13 @@ module KubernetesDeploy
     def record_invalid_template(err:, filename:, content: nil)
       debug_msg = ColorizedString.new("Invalid template: #{filename}\n").red
       debug_msg += "> Error message:\n#{FormattedLogger.indent_four(err)}"
-      debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(content)}" if content
+      if content
+        debug_msg += if content =~ /kind:\s*Secret/
+          "\n> Template content: Suppressed because it may contain a Secret"
+        else
+          "\n> Template content:\n#{FormattedLogger.indent_four(content)}"
+        end
+      end
       @logger.summary.add_paragraph(debug_msg)
     end
 
@@ -445,7 +451,7 @@ module KubernetesDeploy
           prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
         end
 
-        output_is_sensitive = resources.any?(&:kubectl_output_is_sensitive?)
+        output_is_sensitive = resources.any?(&:sensitive_template_content?)
         out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive)
 
         if st.success?
@@ -473,7 +479,7 @@ module KubernetesDeploy
 
       unidentified_errors = []
       filenames_with_sensitive_content = resources
-        .select(&:kubectl_output_is_sensitive?)
+        .select(&:sensitive_template_content?)
         .map { |r| File.basename(r.file_path) }
 
       err.each_line do |line|

data/lib/kubernetes-deploy/ejson_secret_provisioner.rb

@@ -62,7 +62,11 @@ module KubernetesDeploy
 
         secrets.map do |secret_name, secret_spec|
           validate_secret_spec(secret_name, secret_spec)
-          generate_secret_resource(secret_name, secret_spec["_type"], secret_spec["data"])
+          resource = generate_secret_resource(secret_name, secret_spec["_type"], secret_spec["data"])
+          unless resource.validate_definition(@kubectl)
+            raise EjsonSecretError, "Resulting resource Secret/#{secret_name} failed validation"
+          end
+          resource
         end
       end
     end

data/lib/kubernetes-deploy/kubernetes_resource.rb

@@ -30,15 +30,13 @@ module KubernetesDeploy
 
     TIMEOUT_OVERRIDE_ANNOTATION = "kubernetes-deploy.shopify.io/timeout-override"
     LAST_APPLIED_ANNOTATION = "kubectl.kubernetes.io/last-applied-configuration"
-    KUBECTL_OUTPUT_IS_SENSITIVE = false
+    SENSITIVE_TEMPLATE_CONTENT = false
 
     class << self
       def build(namespace:, context:, definition:, logger:, statsd_tags:, crd: nil)
+        validate_definition_essentials(definition)
         opts = { namespace: namespace, context: context, definition: definition, logger: logger,
                  statsd_tags: statsd_tags }
-        if definition["kind"].blank?
-          raise InvalidTemplateError.new("Template missing 'Kind'", content: definition.to_yaml)
-        end
         if (klass = class_for_kind(definition["kind"]))
           return klass.new(**opts)
         end
@@ -53,7 +51,7 @@ module KubernetesDeploy
 
       def class_for_kind(kind)
         if KubernetesDeploy.const_defined?(kind)
-          KubernetesDeploy.const_get(kind) # rubocop:disable Sorbet/ConstantsFromStrings
+          KubernetesDeploy.const_get(kind)
         end
       rescue NameError
         nil
@@ -66,6 +64,24 @@ module KubernetesDeploy
       def kind
         name.demodulize
       end
+
+      private
+
+      def validate_definition_essentials(definition)
+        debug_content = <<~STRING
+          apiVersion: #{definition.fetch('apiVersion', '<missing>')}
+          kind: #{definition.fetch('kind', '<missing>')}
+          metadata: #{definition.fetch('metadata', {})}
+          <Template body suppressed because content sensitivity could not be determined.>
+        STRING
+        if definition["kind"].blank?
+          raise InvalidTemplateError.new("Template is missing required field 'kind'", content: debug_content)
+        end
+
+        if definition.dig('metadata', 'name').blank?
+          raise InvalidTemplateError.new("Template is missing required field 'metadata.name'", content: debug_content)
+        end
+      end
     end
 
     def timeout
@@ -86,12 +102,7 @@ module KubernetesDeploy
 
     def initialize(namespace:, context:, definition:, logger:, statsd_tags: [])
       # subclasses must also set these if they define their own initializer
-      @name = definition.dig("metadata", "name")
-      unless @name.present?
-        logger.summary.add_paragraph("Rendered template content:\n#{definition.to_yaml}")
-        raise FatalDeploymentError, "Template is missing required field metadata.name"
-      end
-
+      @name = definition.dig("metadata", "name").to_s
       @optional_statsd_tags = statsd_tags
       @namespace = namespace
       @context = context
@@ -113,9 +124,9 @@ module KubernetesDeploy
       validate_timeout_annotation
 
       command = ["create", "-f", file_path, "--dry-run", "--output=name"]
-      _, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: kubectl_output_is_sensitive?)
+      _, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: sensitive_template_content?)
       return true if st.success?
-      if kubectl_output_is_sensitive?
+      if sensitive_template_content?
         @validation_errors << <<-EOS
           Validation for #{id} failed. Detailed information is unavailable as the raw error may contain sensitive data.
         EOS
@@ -322,8 +333,8 @@ module KubernetesDeploy
       end
     end
 
-    def kubectl_output_is_sensitive?
-      self.class::KUBECTL_OUTPUT_IS_SENSITIVE
+    def sensitive_template_content?
+      self.class::SENSITIVE_TEMPLATE_CONTENT
     end
 
     class Event

data/lib/kubernetes-deploy/kubernetes_resource/cloudsql.rb

@@ -24,7 +24,7 @@ module KubernetesDeploy
     private
 
     def proxy_deployment_ready?
-      return false unless status = @proxy_deployment["status"]
+      return false unless (status = @proxy_deployment["status"])
       # all cloudsql-proxy pods are running
       status.fetch("availableReplicas", -1) == status.fetch("replicas", 0)
     end

data/lib/kubernetes-deploy/kubernetes_resource/memcached.rb

@@ -26,7 +26,7 @@ module KubernetesDeploy
     private
 
     def deployment_ready?
-      return false unless status = @deployment["status"]
+      return false unless (status = @deployment["status"])
       status.fetch("availableReplicas", -1) == status.fetch("replicas", 0)
     end
 

data/lib/kubernetes-deploy/kubernetes_resource/pod.rb

@@ -146,7 +146,7 @@ module KubernetesDeploy
     end
 
     def ready?
-      return false unless status_data = @instance_data["status"]
+      return false unless (status_data = @instance_data["status"])
       ready_condition = status_data.fetch("conditions", []).find { |condition| condition["type"] == "Ready" }
       ready_condition.present? && (ready_condition["status"] == "True")
     end
@@ -210,8 +210,7 @@ module KubernetesDeploy
         elsif limbo_reason == "CrashLoopBackOff"
           exit_code = @status.dig('lastState', 'terminated', 'exitCode')
           "Crashing repeatedly (exit #{exit_code}). See logs for more information."
-        elsif %w(ImagePullBackOff ErrImagePull).include?(limbo_reason) &&
-          limbo_message.match(/(?:not found)|(?:back-off)/i)
+        elsif limbo_reason == "ErrImagePull" && limbo_message.match(/not found/i)
           "Failed to pull image #{@image}. "\
           "Did you wait for it to be built and pushed to the registry before deploying?"
         elsif limbo_reason == "CreateContainerConfigError"

data/lib/kubernetes-deploy/kubernetes_resource/redis.rb

@@ -29,7 +29,7 @@ module KubernetesDeploy
     private
 
     def deployment_ready?
-      return false unless status = @deployment["status"]
+      return false unless (status = @deployment["status"])
       # all redis pods are running
       status.fetch("availableReplicas", -1) == status.fetch("replicas", 0)
     end

data/lib/kubernetes-deploy/kubernetes_resource/secret.rb

@@ -2,7 +2,7 @@
 module KubernetesDeploy
   class Secret < KubernetesResource
     TIMEOUT = 30.seconds
-    KUBECTL_OUTPUT_IS_SENSITIVE = true
+    SENSITIVE_TEMPLATE_CONTENT = true
 
     def status
       exists? ? "Available" : "Not Found"

data/lib/kubernetes-deploy/resource_cache.rb

@@ -51,7 +51,7 @@ module KubernetesDeploy
 
     def fetch_by_kind(kind)
       resource_class = KubernetesResource.class_for_kind(kind)
-      output_is_sensitive = resource_class.nil? ? false : resource_class::KUBECTL_OUTPUT_IS_SENSITIVE
+      output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT
       raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json",
          output_is_sensitive: output_is_sensitive)
       raise KubectlError unless st.success?

data/lib/kubernetes-deploy/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module KubernetesDeploy
-  VERSION = "0.26.3"
+  VERSION = "0.26.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kubernetes-deploy
 version: !ruby/object:Gem::Version
-  version: 0.26.3
+  version: 0.26.4
 platform: ruby
 authors:
 - Katrina Verey
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-22 00:00:00.000000000 Z
+date: 2019-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -107,14 +107,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.7'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement