krane 2.1.5 → 2.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae99732594c24143e48be0c4c44fdfec9f3f81e7d604362f8d000df837758aaa
-  data.tar.gz: 11684058ac036a9846e5101944c0f26e33216da3d87bde0f20fe13246e52bbb5
+  metadata.gz: 41b26bfb1a0aee84de5462ef6f2e9421e208ead40437e1175c5fffcbbee10a65
+  data.tar.gz: bd21cd0c3ac2c5eeebb75883bb671da5847561e85173a0ffbf40f0b420ff0b57
 SHA512:
-  metadata.gz: 36fec016df1d46e0bec3b7889913bbaa20bdb411795bb58b9995a87dde2bd0ed94203b21d229673fa8fe3d663d60981ce0e4d666acc173e7262b67a3671209ac
-  data.tar.gz: df91b23d97e65489476c5ddb9f404e07c7a314003d84debd6cb5d59c27328f9a48394876af32a7ccb3976d3bfb296f60cd54af4c4fbdca6c31527ce86c2ad19a
+  metadata.gz: 8d53a59534ce296455aaf8b964aec3c0af378da315e7f1d45d56d8f9cb141f9992a72a594146e048487945e17551c6fddb62a46bc52244c8f7af49b5d83c0f88
+  data.tar.gz: 9a2764c120e71f0b429ea2c764ac56bbbedf91d42da5b94a97686809d33025fe81049692b3290970a12768402d2a69891443128b882aa10aafe0d179b6a250ae

data/.rubocop.yml

@@ -1,5 +1,5 @@
-inherit_from:
-  - http://shopify.github.io/ruby-style-guide/rubocop.yml
+inherit_gem:
+  rubocop-shopify: rubocop.yml
 
 AllCops:
   TargetRubyVersion: 2.4

data/.shopify-build/krane.yml

@@ -8,6 +8,14 @@ steps:
     run:
       - bundle: ~
       - bundle exec rubocop
+  - label: 'Run Test Suite (:kubernetes: 1.21-latest :ruby: 3.0)'
+    command: bin/ci
+    agents:
+      queue: k8s-ci
+    env:
+      LOGGING_LEVEL: "4"
+      KUBERNETES_VERSION: v1.21-latest
+      RUBY_VERSION: "3.0"
   - label: 'Run Test Suite (:kubernetes: 1.20-latest :ruby: 3.0)'
     command: bin/ci
     agents:
@@ -38,17 +46,3 @@ steps:
     env:
       LOGGING_LEVEL: "4"
       KUBERNETES_VERSION: v1.17-latest
-  - label: 'Run Test Suite (:kubernetes: 1.16-latest)'
-    command: bin/ci
-    agents:
-      queue: k8s-ci
-    env:
-      LOGGING_LEVEL: "4"
-      KUBERNETES_VERSION: v1.16-latest
-  - label: 'Run Test Suite (:kubernetes: 1.15-latest)'
-    command: bin/ci
-    agents:
-      queue: k8s-ci
-    env:
-      LOGGING_LEVEL: "4"
-      KUBERNETES_VERSION: v1.15-latest

data/CHANGELOG.md

@@ -1,5 +1,44 @@
 ## next
 
+## 2.1.10
+
+*Bug Fixes*
+
+- Don't gather prunable resources by calling uniq only on `kind`: use `group` as well. Otherwise certain resources may not be added to the prune whitelist if the same kind exists across multiple groups [#825](https://github.com/Shopify/krane/pull/825)
+- Fix resource discovery failures when API paths are not located at the root of the API server (this occurs, for example, when using Rancher proxy) [#827](https://github.com/Shopify/krane/pull/827)
+
+*Other*
+
+- Fix ERB deprecation of positional arguments [#828](https://github.com/Shopify/krane/pull/828)
+
+## 2.1.9
+
+*Other*
+
+- Don't package screenshots in the built gem to reduce size [#817](https://github.com/Shopify/krane/pull/817)
+
+## 2.1.8
+
+*Other*
+
+- Change `statsd-instrument` dependency constraint to `< 4` [#815](https://github.com/Shopify/krane/pull/815)
+
+## 2.1.7
+
+*Enhancements*
+- ENV["KRANE_LOG_LINE_LIMIT"] allows the number of container logs printed for failures to be configurable from the 25 line default [#803](https://github.com/Shopify/krane/pull/803).
+
+*Other*
+- Remove the overly tight timeout on cluster resource discovery, which was causing too many timeouts in high latency environments [#813](https://github.com/Shopify/krane/pull/813)
+
+## 2.1.6
+
+*Enhancements*
+- Remove the need for a hard coded GVK overide list via improvements to cluster discovery [#778](https://github.com/Shopify/krane/pull/778)
+
+*Bug Fixes*
+- Remove resources that are targeted by side-effect-inducing mutating admission webhooks from the serverside dry run batch [#798](https://github.com/Shopify/krane/pull/798)
+
 ## 2.1.5
 
 - Fix bug where the wrong dry-run flag is used for kubectl if client version is below 1.18 AND server version is 1.18+ [#793](https://github.com/Shopify/krane/pull/793).

data/dev.yml

@@ -4,7 +4,7 @@ up:
   - ruby: 2.6.6 # Matches gemspec
   - bundler
   - homebrew:
-    - homebrew/cask/minikube
+    - minikube
     - hyperkit
   - custom:
       name: Install the minikube fork of driver-hyperkit

data/krane.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
 
   spec.files         = %x(git ls-files -z).split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
+    f.match(%r{^(test|spec|features|screenshots)/})
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency("googleauth", "~> 0.8")
   spec.add_dependency("ejson", "~> 1.0")
   spec.add_dependency("colorize", "~> 0.8")
-  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 3.1"])
+  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 4"])
   spec.add_dependency("oj", "~> 3.0")
   spec.add_dependency("concurrent-ruby", "~> 1.1")
   spec.add_dependency("jsonpath", "~> 0.9.6")
@@ -57,5 +57,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("ruby-prof")
   spec.add_development_dependency("ruby-prof-flamegraph")
   spec.add_development_dependency("rubocop", "~> 0.89.1")
+  spec.add_development_dependency("rubocop-shopify", "~> 1.0.5")
   spec.add_development_dependency("simplecov")
 end

data/lib/krane/cluster_resource_discovery.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'concurrent'
 
 module Krane
   class ClusterResourceDiscovery
@@ -7,6 +8,7 @@ module Krane
     def initialize(task_config:, namespace_tags: [])
       @task_config = task_config
       @namespace_tags = namespace_tags
+      @api_path_cache = {}
     end
 
     def crds
@@ -18,104 +20,85 @@ module Krane
 
     def prunable_resources(namespaced:)
       black_list = %w(Namespace Node ControllerRevision)
-      api_versions = fetch_api_versions
-
-      fetch_resources(namespaced: namespaced).uniq { |r| r['kind'] }.map do |resource|
-        next unless resource['verbs'].one? { |v| v == "delete" }
-        next if black_list.include?(resource['kind'])
-        gvk_string(api_versions, resource)
+      fetch_resources(namespaced: namespaced).map do |resource|
+        next unless resource["verbs"].one? { |v| v == "delete" }
+        next if black_list.include?(resource["kind"])
+        [resource["apigroup"], resource["version"], resource["kind"]].compact.join("/")
       end.compact
     end
 
-    # kubectl api-resources -o wide returns 5 columns
-    # NAME SHORTNAMES APIGROUP NAMESPACED KIND VERBS
-    # SHORTNAMES and APIGROUP may be blank
-    # VERBS is an array
-    # serviceaccounts sa <blank> true ServiceAccount [create delete deletecollection get list patch update watch]
     def fetch_resources(namespaced: false)
-      command = %w(api-resources)
-      command << "--namespaced=#{namespaced}"
-      raw, err, st = kubectl.run(*command, output: "wide", attempts: 5,
-        use_namespace: false)
-      if st.success?
-        rows = raw.split("\n")
-        header = rows[0]
-        resources = rows[1..-1]
-        full_width_field_names = header.downcase.scan(/[a-z]+[\W]*/)
-        cursor = 0
-        fields = full_width_field_names.each_with_object({}) do |name, hash|
-          start = cursor
-          cursor = start + name.length
-          # Last field should consume the remainder of the line
-          cursor = 0 if full_width_field_names.last == name.strip
-          hash[name.strip] = [start, cursor - 1]
-        end
-        resources.map do |resource|
-          resource = fields.map { |k, (s, e)| [k.strip, resource[s..e].strip] }.to_h
-          # Manually parse verbs: "[get list]" into %w(get list)
-          resource["verbs"] = resource["verbs"][1..-2].split
-          resource
-        end
-      else
-        raise FatalKubeAPIError, "Error retrieving api-resources: #{err}"
+      responses = Concurrent::Hash.new
+      Krane::Concurrency.split_across_threads(api_paths) do |path|
+        responses[path] = fetch_api_path(path)["resources"] || []
       end
+      responses.flat_map do |path, resources|
+        resources.map { |r| resource_hash(path, namespaced, r) }
+      end.compact.uniq { |r| "#{r['apigroup']}/#{r['kind']}" }
     end
 
-    private
-
-    # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
-    # A kind may not exist in all versions of the group.
-    def fetch_api_versions
-      raw, err, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
-      # The "core" group is represented by an empty string
-      versions = { "" => %w(v1) }
+    def fetch_mutating_webhook_configurations
+      command = %w(get mutatingwebhookconfigurations)
+      raw_json, err, st = kubectl.run(*command, output: "json", attempts: 5, use_namespace: false)
       if st.success?
-        rows = raw.split("\n")
-        rows.each do |group_version|
-          group, version = group_version.split("/")
-          versions[group] ||= []
-          versions[group] << version
+        JSON.parse(raw_json)["items"].map do |definition|
+          Krane::MutatingWebhookConfiguration.new(namespace: namespace, context: context, logger: logger,
+            definition: definition, statsd_tags: @namespace_tags)
         end
       else
-        raise FatalKubeAPIError, "Error retrieving api-versions: #{err}"
+        raise FatalKubeAPIError, "Error retrieving mutatingwebhookconfigurations: #{err}"
       end
-      versions
     end
 
-    def version_for_kind(versions, kind)
-      # Override list for kinds that don't appear in the lastest version of a group
-      version_override = { "CronJob" => "v1beta1", "VolumeAttachment" => "v1beta1",
-                           "CSIDriver" => "v1beta1", "Ingress" => "v1beta1",
-                           "CSINode" => "v1beta1", "Job" => "v1",
-                           "IngressClass" => "v1beta1", "FrontendConfig" => "v1beta1",
-                           "ServiceNetworkEndpointGroup" => "v1beta1",
-                           "EnvoyFilter" => "v1alpha3",
-                           "TCPIngress" => "v1beta1" }
+    private
 
-      pattern = /v(?<major>\d+)(?<pre>alpha|beta)?(?<minor>\d+)?/
-      latest = versions.sort_by do |version|
-        match = version.match(pattern)
-        pre = { "alpha" => 0, "beta" => 1, nil => 2 }.fetch(match[:pre])
-        [match[:major].to_i, pre, match[:minor].to_i]
-      end.last
-      version_override.fetch(kind, latest)
-    end
+    # During discovery, the api paths may not actually be at the root, so we must programatically find it.
+    def base_api_path
+      @base_api_path ||= begin
+        raw_response, err, st = kubectl.run("config", "view", "--minify", "--output",
+          "jsonpath={.clusters[*].cluster.server}", attempts: 5, use_namespace: false)
+        raise FatalKubeAPIError, "Error retrieving cluster url: #{err}" unless st.success?
 
-    def gvk_string(api_versions, resource)
-      apiversion = resource['apiversion'].to_s
+        URI(raw_response).path.blank? ? "/" : URI(raw_response).path
+      end
+    end
 
-      ## In kubectl 1.20 APIGroups was replaced by APIVersions
-      if apiversion.empty?
-        apigroup = resource['apigroup'].to_s
-        group_versions = api_versions[apigroup]
+    def api_paths
+      @api_path_cache["/"] ||= begin
+        raw_json, err, st = kubectl.run("get", "--raw", base_api_path, attempts: 5, use_namespace: false)
+        paths = if st.success?
+          JSON.parse(raw_json)["paths"]
+        else
+          raise FatalKubeAPIError, "Error retrieving raw path /: #{err}"
+        end
+        paths.select { |path| %r{^\/api.*\/v.*$}.match(path) }
+      end
+    end
 
-        version = version_for_kind(group_versions, resource['kind'])
-        apigroup = 'core' if apigroup.empty?
-        apiversion = "#{apigroup}/#{version}"
+    def fetch_api_path(path)
+      @api_path_cache[path] ||= begin
+        raw_json, err, st = kubectl.run("get", "--raw", path, attempts: 2, use_namespace: false)
+        if st.success?
+          JSON.parse(raw_json)
+        else
+          logger.warn("Error retrieving api path: #{err}")
+          {}
+        end
       end
+    end
 
-      apiversion = "core/#{apiversion}" unless apiversion.include?("/")
-      [apiversion, resource['kind']].compact.join("/")
+    def resource_hash(path, namespaced, blob)
+      return unless blob["namespaced"] == namespaced
+      # skip sub-resources
+      return if blob["name"].include?("/")
+      path_regex = %r{(/apis?/)(?<group>[^/]*)/?(?<version>v.+)}
+      match = path.match(path_regex)
+      {
+        "verbs" => blob["verbs"],
+        "kind" => blob["kind"],
+        "apigroup" => match[:group],
+        "version" => match[:version],
+      }
     end
 
     def fetch_crds

data/lib/krane/container_logs.rb

@@ -3,7 +3,7 @@ module Krane
   class ContainerLogs
     attr_reader :lines, :container_name
 
-    DEFAULT_LINE_LIMIT = 25
+    DEFAULT_LINE_LIMIT = Integer(ENV.fetch('KRANE_LOG_LINE_LIMIT', 25))
 
     def initialize(parent_id:, container_name:, namespace:, context:, logger:)
       @parent_id = parent_id

data/lib/krane/deploy_task.rb

@@ -30,6 +30,7 @@ require 'krane/kubernetes_resource'
   custom_resource_definition
   horizontal_pod_autoscaler
   secret
+  mutating_webhook_configuration
 ).each do |subresource|
   require "krane/kubernetes_resource/#{subresource}"
 end
@@ -277,16 +278,25 @@ module Krane
     end
     measure_method(:validate_configuration)
 
+    def partition_dry_run_resources(resources)
+      individuals = []
+      mutating_webhook_configurations = cluster_resource_discoverer.fetch_mutating_webhook_configurations
+      mutating_webhook_configurations.each do |mutating_webhook_configuration|
+        mutating_webhook_configuration.webhooks.each do |webhook|
+          individuals = (individuals + resources.select { |resource| webhook.matches_resource?(resource) }).uniq
+          resources -= individuals
+        end
+      end
+      [resources, individuals]
+    end
+
     def validate_resources(resources)
       validate_globals(resources)
-      batch_dry_run_success = kubectl.server_dry_run_enabled? && validate_dry_run(resources)
+      batchable_resources, individuals = partition_dry_run_resources(resources.dup)
+      batch_dry_run_success = kubectl.server_dry_run_enabled? && validate_dry_run(batchable_resources)
+      individuals += batchable_resources unless batch_dry_run_success
       Krane::Concurrency.split_across_threads(resources) do |r|
-        # No need to pass in kubectl (and do per-resource dry run apply) if batch dry run succeeded
-        if batch_dry_run_success
-          r.validate_definition(kubectl: nil, selector: @selector, dry_run: false)
-        else
-          r.validate_definition(kubectl: kubectl, selector: @selector, dry_run: true)
-        end
+        r.validate_definition(kubectl: kubectl, selector: @selector, dry_run: individuals.include?(r))
       end
       failed_resources = resources.select(&:validation_failed?)
       if failed_resources.present?

data/lib/krane/kubernetes_resource.rb

@@ -226,6 +226,11 @@ module Krane
       version ? grouping : "core"
     end
 
+    def version
+      prefix, version = @definition.dig("apiVersion").split("/")
+      version || prefix
+    end
+
     def kubectl_resource_type
       type
     end

data/lib/krane/kubernetes_resource/mutating_webhook_configuration.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Krane
+  class MutatingWebhookConfiguration < KubernetesResource
+    GLOBAL = true
+
+    class Webhook
+      EQUIVALENT = 'Equivalent'
+      EXACT = 'Exact'
+
+      class Rule
+        def initialize(definition)
+          @definition = definition
+        end
+
+        def matches_resource?(resource, accept_equivalent:)
+          groups.each do |group|
+            versions.each do |version|
+              resources.each do |kind|
+                return true if (resource.group == group || group == '*' || accept_equivalent) &&
+                  (resource.version == version || version == '*' || accept_equivalent) &&
+                  (resource.type.downcase == kind.downcase.singularize || kind == "*")
+              end
+            end
+          end
+          false
+        end
+
+        def groups
+          @definition.dig('apiGroups')
+        end
+
+        def versions
+          @definition.dig('apiVersions')
+        end
+
+        def resources
+          @definition.dig('resources')
+        end
+      end
+
+      def initialize(definition)
+        @definition = definition
+      end
+
+      def side_effects
+        @definition.dig('sideEffects')
+      end
+
+      def has_side_effects?
+        !%w(None NoneOnDryRun).include?(side_effects)
+      end
+
+      def match_policy
+        @definition.dig('matchPolicy')
+      end
+
+      def matches_resource?(resource, skip_rule_if_side_effect_none: true)
+        return false if skip_rule_if_side_effect_none && !has_side_effects?
+        rules.any? do |rule|
+          rule.matches_resource?(resource, accept_equivalent: match_policy == EQUIVALENT)
+        end
+      end
+
+      def rules
+        @definition.fetch('rules', []).map { |rule| Rule.new(rule) }
+      end
+    end
+
+    def initialize(namespace:, context:, definition:, logger:, statsd_tags:)
+      @webhooks = (definition.dig('webhooks') || []).map { |hook| Webhook.new(hook) }
+      super(namespace: namespace, context: context, definition: definition,
+        logger: logger, statsd_tags: statsd_tags)
+    end
+
+    TIMEOUT = 30.seconds
+
+    def deploy_succeeded?
+      exists?
+    end
+
+    def webhooks
+      @definition.fetch('webhooks', []).map { |webhook| Webhook.new(webhook) }
+    end
+  end
+end

data/lib/krane/renderer.rb

@@ -39,7 +39,7 @@ module Krane
       erb_binding = TemplateContext.new(self).template_binding
       bind_template_variables(erb_binding, template_variables)
 
-      ERB.new(raw_template, nil, '-').result(erb_binding)
+      ERB.new(raw_template, trim_mode: '-').result(erb_binding)
     rescue InvalidPartialError => err
       err.parents = err.parents.dup.unshift(filename)
       err.filename = "#{err.filename} (partial included from: #{err.parents.join(' -> ')})"
@@ -56,7 +56,7 @@ module Krane
 
       partial_path = find_partial(partial)
       template = File.read(partial_path)
-      expanded_template = ERB.new(template, nil, '-').result(erb_binding)
+      expanded_template = ERB.new(template, trim_mode: '-').result(erb_binding)
 
       docs = Psych.parse_stream(expanded_template, partial_path)
       # If the partial contains multiple documents or has an explicit document header,

data/lib/krane/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Krane
-  VERSION = "2.1.5"
+  VERSION = "2.1.10"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krane
 version: !ruby/object:Gem::Version
-  version: 2.1.5
+  version: 2.1.10
 platform: ruby
 authors:
 - Katrina Verey
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-27 00:00:00.000000000 Z
+date: 2021-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -91,7 +91,7 @@ dependencies:
         version: '2.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -101,7 +101,7 @@ dependencies:
         version: '2.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -374,6 +374,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.89.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-shopify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -460,6 +474,7 @@ files:
 - lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb
 - lib/krane/kubernetes_resource/ingress.rb
 - lib/krane/kubernetes_resource/job.rb
+- lib/krane/kubernetes_resource/mutating_webhook_configuration.rb
 - lib/krane/kubernetes_resource/network_policy.rb
 - lib/krane/kubernetes_resource/persistent_volume_claim.rb
 - lib/krane/kubernetes_resource/pod.rb
@@ -493,11 +508,6 @@ files:
 - lib/krane/task_config_validator.rb
 - lib/krane/template_sets.rb
 - lib/krane/version.rb
-- screenshots/deploy-demo.gif
-- screenshots/migrate-logs.png
-- screenshots/missing-secret-fail.png
-- screenshots/success.png
-- screenshots/test-output.png
 homepage: https://github.com/Shopify/krane
 licenses:
 - MIT
@@ -518,7 +528,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: A command line tool that helps you ship changes to a Kubernetes namespace