krane 1.1.2 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78d05d646de5dda6c38cd1fec3e5abca3964b656e7bd432ebb4f9e537dd3a88c
-  data.tar.gz: 41a4ed27fc0a646c4471f4c6505ec24eb8ff605565f86eef1ea78ff75b614f7d
+  metadata.gz: d5cbad8d48ae06d3286c19cd193ef7208c04c74ee72b3519a82a25605d9c4e86
+  data.tar.gz: 2eded80d29111517ca964fe81033d6c3e3667b8704e9c37de3e293fe06fab987
 SHA512:
-  metadata.gz: c8f7269fb7fb49b758bf6ac8127adf93bdcc59417ca6a3789b879e28cde2628e1bfacef8bda9f0b3534acc03792d359e5fbfad70260dab173179db362c007255
-  data.tar.gz: cda59df1d38965783cf146c847db571b697ab23737655744f4123fa7b169f72aa0aaf9305b82f145b68792e1a58cf5b2571427db9cef340864c660b21ae82c61
+  metadata.gz: bf04a253ca03e3b3741e7c57dfd3967f54ccfc22b1094da4e9ca854e8211ce0f2fff4c1bc8ed56fef3c4fb7aefe63dd7a53c601e374ebdeaaf6274749722c7ec
+  data.tar.gz: db700598c8ec1e634c3c0db7802a6822668651355a2c3c509c1a4ff6dd54947b1999e36ee60404c62b397ffd58287eca93f272497ecfb7114082cdb029410595

data/.rubocop-http---shopify-github-io-ruby-style-guide-rubocop-yml

@@ -1,5 +1,3 @@
-# Recommended rubocop version: ~> 0.78.0
-
 AllCops:
   Exclude:
   - 'db/schema.rb'
@@ -78,13 +76,6 @@ Style/BlockDelimiters:
   - proc
   - it
 
-Style/BracesAroundHashParameters:
-  EnforcedStyle: no_braces
-  SupportedStyles:
-  - braces
-  - no_braces
-  - context_dependent
-
 Layout/CaseIndentation:
   EnforcedStyle: end
   SupportedStyles:
@@ -273,7 +264,7 @@ Style/MethodCallWithArgsParentheses:
   - raise
   - puts
   Exclude:
-  - Gemfile
+  - '**/Gemfile'
 
 Style/MethodDefParentheses:
   EnforcedStyle: require_parentheses
@@ -586,6 +577,7 @@ Layout/BlockEndNewline:
 
 Style/CaseEquality:
   Enabled: true
+  AllowOnConstant: true
 
 Style/CharacterLiteral:
   Enabled: true
@@ -668,6 +660,9 @@ Style/IfWithSemicolon:
 Style/IdenticalConditionalBranches:
   Enabled: true
 
+Layout/IndentationStyle:
+  Enabled: true
+
 Style/InfiniteLoop:
   Enabled: true
 
@@ -680,7 +675,7 @@ Style/LineEndConcatenation:
 Style/MethodCallWithoutArgsParentheses:
   Enabled: true
 
-Style/MethodMissingSuper:
+Lint/MissingSuper:
   Enabled: true
 
 Style/MissingRespondToMissing:
@@ -812,9 +807,6 @@ Layout/SpaceInsideRangeLiteral:
 Style/SymbolLiteral:
   Enabled: true
 
-Layout/Tab:
-  Enabled: true
-
 Layout/TrailingWhitespace:
   Enabled: true
 
@@ -843,7 +835,7 @@ Style/ZeroLengthPredicate:
   Enabled: true
 
 Layout/HeredocIndentation:
-  EnforcedStyle: squiggly
+  Enabled: true
 
 Lint/AmbiguousOperator:
   Enabled: true
@@ -881,9 +873,6 @@ Lint/EmptyEnsure:
 Lint/EmptyInterpolation:
   Enabled: true
 
-Lint/EndInMethod:
-  Enabled: true
-
 Lint/EnsureReturn:
   Enabled: true
 
@@ -976,7 +965,7 @@ Lint/UselessAccessModifier:
 Lint/UselessAssignment:
   Enabled: true
 
-Lint/UselessComparison:
+Lint/BinaryOperatorWithIdenticalOperands:
   Enabled: true
 
 Lint/UselessElseWithoutRescue:
@@ -1027,3 +1016,6 @@ Style/ModuleFunction:
 
 Lint/OrderedMagicComments:
   Enabled: true
+
+Lint/DeprecatedOpenSSLConstant:
+  Enabled: true

data/.shopify-build/krane.yml

@@ -1,21 +1,36 @@
 containers:
   default:
-    docker: circleci/ruby:2.4.6-node-browsers
+    docker: circleci/ruby:2.5.7
 
 steps:
   - label: Lint
     timeout: 5m
     run:
+      - bundle: ~
       - bundle exec rubocop
-    dependencies:
-      - bundler
-  - label: 'Run Test Suite (:kubernetes: 1.16-latest)'
+  - label: 'Run Test Suite (:kubernetes: 1.18-latest :ruby: 2.7)'
     command: bin/ci
     agents:
       queue: k8s-ci
     env:
       LOGGING_LEVEL: "4"
-      KUBERNETES_VERSION: v1.16-latest
+      KUBERNETES_VERSION: v1.18-latest
+      RUBY_VERSION: "2.7"
+  - label: 'Run Test Suite (:kubernetes: 1.17-latest)'
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: "4"
+      KUBERNETES_VERSION: v1.17-latest
+  - label: 'Run Test Suite (:kubernetes: 1.16.12)'
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: "4"
+      # Flip this back to v1.16-latest when 1.16.14 comes out (see #733)
+      KUBERNETES_VERSION: v1.16.12
   - label: 'Run Test Suite (:kubernetes: 1.15-latest)'
     command: bin/ci
     agents:
@@ -51,4 +66,3 @@ steps:
     env:
       LOGGING_LEVEL: "4"
       KUBERNETES_VERSION: v1.11-latest
-

data/1.0-Upgrade.md

@@ -119,7 +119,7 @@ Old flag | New flag | Comments
 --skip-wait | --verify-result=true |
 --allow-protected-ns | --protected-namespaces=default,kube-system,kube-public | Added the ability to specify which namespaces are protected
 --no-prune | --prune=true |
---template-dir | -f, --filename | Makes all krane commands accept this argument, which is now required for the deploy task
+--template-dir | -f, --filenames | Makes all krane commands accept this argument, which is now required for the deploy task
 --verbose-log-prefix | --verbose-log-prefix |
 --max-watch-seconds=seconds | --global-timeout=300s | Changed flag name and default value to be a duration (expressed using strings like "300s" or "1h")
 --selector | --selector |

data/CHANGELOG.md

@@ -1,6 +1,47 @@
 ## next
 
-...
+## 2.1.1
+
+*Bug Fixes*
+- Fix the way environment variables are passed into the EJSON decryption invocation [#759](https://github.com/Shopify/krane/pull/759)
+
+## 2.1.0
+
+*Features*
+- _(experimental)_ Override deploy method via annotation. This feature is considered alpha and should not be considered stable [#753](https://github.com/Shopify/krane/pull/753)
+
+*Enhancements*
+- Increased the number of attempts on kubectl commands during Initializing deploy phase [#749](https://github.com/Shopify/krane/pull/749)
+- Increased attempts on kubectl apply command during deploy [#751](https://github.com/Shopify/krane/pull/751)
+- Whitelist context deadline error during kubctl dry run [#754](https://github.com/Shopify/krane/pull/754)
+- Allow specifying a kubeconfig per task in the internal API [#746](https://github.com/Shopify/krane/pull/746)
+
+## 2.0.0
+
+*Breaking Changes*
+- Remove kubernetes deploy annotation prefix [#738](https://github.com/Shopify/krane/pull/738)
+
+*Bug Fixes*
+- Always set a deployment_id, even if the current_sha isn't set [#730](https://github.com/Shopify/krane/pull/730)
+- YAML string scalars with scientific/e-notation numeric format not properly quoted. [#740](https://github.com/Shopify/krane/pull/740)
+
+## 1.1.4
+
+*Bug Fixes*
+- Properly look up constant on Krane namespace. [#720](https://github.com/Shopify/krane/pull/720)
+
+*Enhancements*
+- Allow to configure `image_tag` when using task runner. [#719](https://github.com/Shopify/krane/pull/719)
+
+## 1.1.3
+
+*Bug Fixes*
+- Retry dry-run validation when no error is returned. [#705](https://github.com/Shopify/krane/pull/705)
+- Stop deploys if ClusterResourceDiscovery's kubectl calls fail. [#701](https://github.com/Shopify/krane/pull/701)
+
+*Other*
+- Dropped support for Ruby 2.4 since it will be EoL shortly. [#693](https://github.com/Shopify/krane/pull/693).
+- Ruby 2.7 support: fix deprecation warnings, add testing. [#710](https://github.com/Shopify/krane/pull/705)
 
 ## 1.1.2
 *Enhancements*
@@ -43,6 +84,12 @@
 We've renamed the gem and cli to Krane.
 See our [migration guide](https://github.com/Shopify/krane/blob/master/1.0-Upgrade.md) to help navigate the breaking changes.
 
+## 1.0.0.pre.2
+
+*Enhancements*
+- Relax thor version requirement. ([#731](https://github.com/Shopify/krane/pull/731))
+- Relax googleauth restriction. ([#731](https://github.com/Shopify/krane/pull/731))
+
 ## 1.0.0.pre.1
 
 *Important!*

data/CONTRIBUTING.md

@@ -151,7 +151,7 @@ To see the full-color output of a specific integration test, you can use `PRINT_
 1. Make sure CHANGELOG.md includes all user-facing changes since the last release. Things like test changes or refactors do not need to be included.
 1. Update the version number in `version.rb`.
 1. Commit your changes with message "Version x.y.z" and open a PR.
-1. After merging your PR, deploy via [Shipit](https://shipit.shopify.io/shopify/kubernetes-deploy/rubygems). Shipit will automatically tag the release and upload the gem to [rubygems.org](https://rubygems.org/gems/krane).
+1. After merging your PR, deploy via [Shipit](https://shipit.shopify.io/shopify/krane/rubygems). Shipit will automatically tag the release and upload the gem to [rubygems.org](https://rubygems.org/gems/krane).
 
 ## CI (External contributors)
 

data/README.md

@@ -73,7 +73,7 @@ If you need the ability to render dynamic values in templates before deploying,
 
 ## Prerequisites
 
-* Ruby 2.4+
+* Ruby 2.5+
 * Your cluster must be running Kubernetes v1.11.0 or higher<sup>1</sup>
 
 <sup>1</sup> We run integration tests against these Kubernetes versions. You can find our
@@ -103,7 +103,7 @@ official compatibility chart below.
 
 *Environment variables:*
 
-- `$KUBECONFIG`: points to one or multiple valid kubeconfig files that include the context you want to deploy to. File names are separated by colon for Linux and Mac, and semi-colon for Windows. If ommitted, will use the Kubernetes default of `~/.kube/config`.
+- `$KUBECONFIG`: points to one or multiple valid kubeconfig files that include the context you want to deploy to. File names are separated by colon for Linux and Mac, and semi-colon for Windows. If omitted, Krane will use the Kubernetes default of `~/.kube/config`.
 - `$GOOGLE_APPLICATION_CREDENTIALS`: points to the credentials for an authenticated service account (required if your kubeconfig `user`'s auth provider is GCP)
 
 
@@ -159,6 +159,11 @@ before the deployment is considered successful.
   - _Default_: `true`
   - `true`: The custom resource will be deployed in the pre-deploy phase.
   - All other values: The custom resource will be deployed in the main deployment phase.
+- `krane.shopify.io/deploy-method-override`: Cause a resource to be deployed by the specified `kubectl` command, instead of the default `apply`.
+  - _Compatibility_: Cannot be used for `PodDisruptionBudget`, since it always uses `create/replace-force`
+  - _Accepted values_: `create`, `replace`, and `replace-force`
+  - _Warning_: Resources whose deploy method is overridden are no longer subject to pruning on deploy.
+  - This feature is _experimental_ and may be removed at any time.
 
 
 ### Running tasks at the beginning of a deploy
@@ -544,9 +549,7 @@ krane render -f ./path/to/template/dir/template.yaml.erb > template.yaml
 - `--bindings=BINDINGS`: Makes additional variables available to your ERB templates. For example, `krane render --bindings=color=blue size=large -f some-template.yaml.erb` will expose `color` and `size` to `some-template.yaml.erb`.
 - `--current-sha`: Expose SHA `current_sha` in ERB bindings
 
-You can add additional variables using the `--bindings=BINDINGS` option which can be formated as a string, JSON string or path to a JSON or YAML file. Complex JSON or YAML data will be converted to a Hash for use in templates. To load a file the argument should include the relative
-file path prefixed with an `@` sign. An argument error will be raised if the string argument cannot be parsed, the referenced file does not include a
- valid extension (`.json`, `.yaml` or `.yml`) or the referenced file does not exist.
+You can add additional variables using the `--bindings=BINDINGS` option which can be formatted as a string, JSON string or path to a JSON or YAML file. Complex JSON or YAML data will be converted to a Hash for use in templates. To load a file, the argument should include the relative file path prefixed with an `@` sign. An argument error will be raised if the string argument cannot be parsed, the referenced file does not include a valid extension (`.json`, `.yaml` or `.yml`) or the referenced file does not exist.
 
 #### Bindings examples
 

data/bin/ci

@@ -17,5 +17,5 @@ docker run --rm \
     -e VERBOSE=1 \
     -e PARALLELISM=$PARALLELISM \
     -w /usr/src/app \
-    ruby:2.4 \
+    ruby:"${RUBY_VERSION:-2.5}" \
     bin/test

data/dev.yml

@@ -1,10 +1,11 @@
 ---
 name: krane
 up:
-  - ruby: 2.4.6 # Matches gemspec
+  - ruby: 2.5.7 # Matches gemspec
   - bundler
   - homebrew:
-    - Caskroom/cask/minikube
+    - homebrew/cask/minikube
+    - hyperkit
   - custom:
       name: Install the minikube fork of driver-hyperkit
       met?: command -v docker-machine-driver-hyperkit

data/krane.gemspec

@@ -23,7 +23,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = %w(lib)
 
-  spec.required_ruby_version = '>= 2.4.0'
+  spec.metadata['allowed_push_host'] = "https://rubygems.org"
+
+  spec.required_ruby_version = '>= 2.5.0'
   spec.add_dependency("activesupport", ">= 5.0")
   spec.add_dependency("kubeclient", "~> 4.3")
   spec.add_dependency("googleauth", "~> 0.8")
@@ -54,6 +56,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("byebug")
   spec.add_development_dependency("ruby-prof")
   spec.add_development_dependency("ruby-prof-flamegraph")
-  spec.add_development_dependency("rubocop", "~> 0.78.0")
+  spec.add_development_dependency("rubocop", "~> 0.89.1")
   spec.add_development_dependency("codecov")
 end

data/lib/krane/annotation.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Krane
+  module Annotation
+    class << self
+      def for(suffix)
+        "krane.shopify.io/#{suffix}"
+      end
+    end
+  end
+end

data/lib/krane/cluster_resource_discovery.rb

@@ -37,7 +37,7 @@ module Krane
     def fetch_resources(namespaced: false)
       command = %w(api-resources)
       command << "--namespaced=#{namespaced}"
-      raw, _, st = kubectl.run(*command, output: "wide", attempts: 5,
+      raw, err, st = kubectl.run(*command, output: "wide", attempts: 5,
         use_namespace: false)
       if st.success?
         rows = raw.split("\n")
@@ -59,7 +59,7 @@ module Krane
           resource
         end
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving api-resources: #{err}"
       end
     end
 
@@ -68,7 +68,7 @@ module Krane
     # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
     # A kind may not exist in all versions of the group.
     def fetch_api_versions
-      raw, _, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
+      raw, err, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
       # The "core" group is represented by an empty string
       versions = { "" => %w(v1) }
       if st.success?
@@ -78,6 +78,8 @@ module Krane
           versions[group] ||= []
           versions[group] << version
         end
+      else
+        raise FatalKubeAPIError, "Error retrieving api-versions: #{err}"
       end
       versions
     end
@@ -86,7 +88,8 @@ module Krane
       # Override list for kinds that don't appear in the lastest version of a group
       version_override = { "CronJob" => "v1beta1", "VolumeAttachment" => "v1beta1",
                            "CSIDriver" => "v1beta1", "Ingress" => "v1beta1",
-                           "CSINode" => "v1beta1", "Job" => "v1" }
+                           "CSINode" => "v1beta1", "Job" => "v1",
+                           "IngressClass" => "v1beta1" }
 
       pattern = /v(?<major>\d+)(?<pre>alpha|beta)?(?<minor>\d+)?/
       latest = versions.sort_by do |version|
@@ -98,12 +101,12 @@ module Krane
     end
 
     def fetch_crds
-      raw_json, _, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
+      raw_json, err, st = kubectl.run("get", "CustomResourceDefinition", output: "json", attempts: 5,
         use_namespace: false)
       if st.success?
         JSON.parse(raw_json)["items"]
       else
-        []
+        raise FatalKubeAPIError, "Error retrieving CustomResourceDefinition: #{err}"
       end
     end
 

data/lib/krane/concerns/template_reporting.rb

@@ -15,12 +15,6 @@ module Krane
       logger.summary.add_paragraph(debug_msg)
     end
 
-    def record_warnings(logger:, warning:, filename:)
-      warn_msg = "Template warning: #{filename}\n"
-      warn_msg += "> Warning message:\n#{Krane::FormattedLogger.indent_four(warning)}"
-      logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
-    end
-
     def add_para_from_list(logger:, action:, enum:)
       logger.summary.add_action(action)
       logger.summary.add_paragraph(enum.map { |e| "- #{e}" }.join("\n"))

data/lib/krane/container_overrides.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+module Krane
+  class ContainerOverrides
+    attr_reader :command, :arguments, :env_vars, :image_tag
+
+    def initialize(command: nil, arguments: nil, env_vars: [], image_tag: nil)
+      @command = command
+      @arguments = arguments
+      @env_vars = env_vars
+      @image_tag = image_tag
+    end
+
+    def apply!(container)
+      container.command = command if command
+      container.args = arguments if arguments
+
+      if image_tag
+        image = container.image
+        base_image, _old_tag = image.split(':')
+        new_image = "#{base_image}:#{image_tag}"
+
+        container.image = new_image
+      end
+
+      env_args = env_vars.map do |env|
+        key, value = env.split('=', 2)
+        { name: key, value: value }
+      end
+      container.env ||= []
+      container.env = container.env.map(&:to_h) + env_args
+    end
+  end
+end

data/lib/krane/deploy_task.rb

@@ -4,6 +4,7 @@ require 'shellwords'
 require 'tempfile'
 require 'fileutils'
 
+require 'krane/annotation'
 require 'krane/common'
 require 'krane/concurrency'
 require 'krane/resource_cache'
@@ -56,6 +57,7 @@ module Krane
     )
 
     def predeploy_sequence
+      default_group = { group: nil }
       before_crs = %w(
         ResourceQuota
         NetworkPolicy
@@ -65,12 +67,14 @@ module Krane
         Role
         RoleBinding
         Secret
-      )
+      ).map { |r| [r, default_group] }
+
       after_crs = %w(
         Pod
-      )
+      ).map { |r| [r, default_group] }
 
-      before_crs + cluster_resource_discoverer.crds.select(&:predeployed?).map(&:kind) + after_crs
+      crs = cluster_resource_discoverer.crds.select(&:predeployed?).map { |cr| [cr.kind, { group: cr.group }] }
+      Hash[before_crs + crs + after_crs]
     end
 
     def prune_whitelist
@@ -81,6 +85,10 @@ module Krane
       kubectl.server_version
     end
 
+    attr_reader :task_config
+
+    delegate :kubeclient_builder, to: :task_config
+
     # Initializes the deploy task
     #
     # @param namespace [String] Kubernetes namespace (*required*)
@@ -97,10 +105,10 @@ module Krane
     # @param render_erb [Boolean] Enable ERB rendering
     def initialize(namespace:, context:, current_sha: nil, logger: nil, kubectl_instance: nil, bindings: {},
       global_timeout: nil, selector: nil, filenames: [], protected_namespaces: nil,
-      render_erb: false)
+      render_erb: false, kubeconfig: nil)
       @logger = logger || Krane::FormattedLogger.build(namespace, context)
       @template_sets = TemplateSets.from_dirs_and_files(paths: filenames, logger: @logger, render_erb: render_erb)
-      @task_config = Krane::TaskConfig.new(context, namespace, @logger)
+      @task_config = Krane::TaskConfig.new(context, namespace, @logger, kubeconfig)
       @bindings = bindings
       @namespace = namespace
       @namespace_tags = []
@@ -116,8 +124,8 @@ module Krane
     # Runs the task, returning a boolean representing success or failure
     #
     # @return [Boolean]
-    def run(*args)
-      run!(*args)
+    def run(**args)
+      run!(**args)
       true
     rescue FatalDeploymentError
       false
@@ -186,10 +194,6 @@ module Krane
         selector: @selector, statsd_tags: statsd_tags, current_sha: @current_sha)
     end
 
-    def kubeclient_builder
-      @kubeclient_builder ||= KubeclientBuilder.new
-    end
-
     def cluster_resource_discoverer
       @cluster_resource_discoverer ||= ClusterResourceDiscovery.new(
         task_config: @task_config,
@@ -210,7 +214,10 @@ module Krane
     end
 
     def deploy_has_priority_resources?(resources)
-      resources.any? { |r| predeploy_sequence.include?(r.type) }
+      resources.any? do |r|
+        next unless (pr = predeploy_sequence[r.type])
+        !pr[:group] || pr[:group] == r.group
+      end
     end
 
     def check_initial_status(resources)
@@ -242,6 +249,8 @@ module Krane
         @logger.info("  - #{secret.id} (from ejson)")
       end
 
+      StatsD.client.gauge('discover_resources.count', resources.size, tags: statsd_tags)
+
       resources.sort
     rescue InvalidTemplateError => e
       record_invalid_template(logger: @logger, err: e.message, filename: e.filename,
@@ -274,11 +283,6 @@ module Krane
         r.validate_definition(kubectl, selector: @selector)
       end
 
-      resources.select(&:has_warnings?).each do |resource|
-        record_warnings(logger: @logger, warning: resource.validation_warning_msg,
-          filename: File.basename(resource.file_path))
-      end
-
       failed_resources = resources.select(&:validation_failed?)
       if failed_resources.present?