krane 2.1.4 → 2.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ae8bb4b0c7f920136990847601e19c7767075a89ead99781213feaf309d623d
-  data.tar.gz: 90ad58122ca788ebd4010e62e624c0d73a0c153e48c942673e244e7fbb4ccb32
+  metadata.gz: 6126a210b181939121f8639ea63915439fb4188f0a82220b8c6db53cf7aa4980
+  data.tar.gz: f1808e32d27cae24775e65fa6fbb0ddb47e99b6b0ca1a62456ec721d386b94d8
 SHA512:
-  metadata.gz: 7e09bdb58b4bca4380b77bb10589a2e5b57aa31e9d8838c4b6efc68ccbf58b5af84c360a1a2710564164e4d6f19419cd38d7b3952559c56d5f7dc026d432a194
-  data.tar.gz: 659649bf0d0cfbe04d3fad0f32f3a05cc6ed9e117ca0755bf808d898b60dbf0d5be88cad609de1589cd93eb2b808e6cc9994a73f7b21267920b679b71afd138e
+  metadata.gz: 3fe412af18f0473f2fbb25b03ec34beee430f2d9483884af66e24220b4823059caf0b9b9974ba3bbc62481fe42bea8b86eda4efb341f335e259c631f85dfdbab
+  data.tar.gz: 8cb300e21c0608ad62b3caa3f82fc818e6de7cb11a2c8c8c63607b1b92335f6d0c198916f2b2fabd271646958e64db78a67d3251d492607d2d4301b5b0fe3e35

data/.rubocop.yml

@@ -1,5 +1,5 @@
-inherit_from:
-  - http://shopify.github.io/ruby-style-guide/rubocop.yml
+inherit_gem:
+  rubocop-shopify: rubocop.yml
 
 AllCops:
   TargetRubyVersion: 2.4

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 ## next
 
+## 2.1.9
+
+*Other*
+
+- Don't package screenshots in the built gem to reduce size [#817](https://github.com/Shopify/krane/pull/817)
+
+## 2.1.8
+
+*Other*
+
+- Change `statsd-instrument` dependency constraint to `< 4` [#815](https://github.com/Shopify/krane/pull/815)
+
+## 2.1.7
+
+*Enhancements*
+- ENV["KRANE_LOG_LINE_LIMIT"] allows the number of container logs printed for failures to be configurable from the 25 line default [#803](https://github.com/Shopify/krane/pull/803).
+
+*Other*
+- Remove the overly tight timeout on cluster resource discovery, which was causing too many timeouts in high latency environments [#813](https://github.com/Shopify/krane/pull/813)
+
+## 2.1.6
+
+*Enhancements*
+- Remove the need for a hard coded GVK overide list via improvements to cluster discovery [#778](https://github.com/Shopify/krane/pull/778)
+
+*Bug Fixes*
+- Remove resources that are targeted by side-effect-inducing mutating admission webhooks from the serverside dry run batch [#798](https://github.com/Shopify/krane/pull/798)
+
+## 2.1.5
+
+- Fix bug where the wrong dry-run flag is used for kubectl if client version is below 1.18 AND server version is 1.18+ [#793](https://github.com/Shopify/krane/pull/793).
+
 ## 2.1.4
 
 *Enhancements*

data/dev.yml

@@ -4,7 +4,7 @@ up:
   - ruby: 2.6.6 # Matches gemspec
   - bundler
   - homebrew:
-    - homebrew/cask/minikube
+    - minikube
     - hyperkit
   - custom:
       name: Install the minikube fork of driver-hyperkit

data/krane.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
 
   spec.files         = %x(git ls-files -z).split("\x0").reject do |f|
-    f.match(%r{^(test|spec|features)/})
+    f.match(%r{^(test|spec|features|screenshots)/})
   end
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency("googleauth", "~> 0.8")
   spec.add_dependency("ejson", "~> 1.0")
   spec.add_dependency("colorize", "~> 0.8")
-  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 3.1"])
+  spec.add_dependency("statsd-instrument", ['>= 2.8', "< 4"])
   spec.add_dependency("oj", "~> 3.0")
   spec.add_dependency("concurrent-ruby", "~> 1.1")
   spec.add_dependency("jsonpath", "~> 0.9.6")
@@ -57,5 +57,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency("ruby-prof")
   spec.add_development_dependency("ruby-prof-flamegraph")
   spec.add_development_dependency("rubocop", "~> 0.89.1")
+  spec.add_development_dependency("rubocop-shopify", "~> 1.0.5")
   spec.add_development_dependency("simplecov")
 end

data/lib/krane/cluster_resource_discovery.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'concurrent'
 
 module Krane
   class ClusterResourceDiscovery
@@ -7,6 +8,7 @@ module Krane
     def initialize(task_config:, namespace_tags: [])
       @task_config = task_config
       @namespace_tags = namespace_tags
+      @api_path_cache = {}
     end
 
     def crds
@@ -18,104 +20,74 @@ module Krane
 
     def prunable_resources(namespaced:)
       black_list = %w(Namespace Node ControllerRevision)
-      api_versions = fetch_api_versions
-
-      fetch_resources(namespaced: namespaced).uniq { |r| r['kind'] }.map do |resource|
-        next unless resource['verbs'].one? { |v| v == "delete" }
-        next if black_list.include?(resource['kind'])
-        gvk_string(api_versions, resource)
+      fetch_resources(namespaced: namespaced).map do |resource|
+        next unless resource["verbs"].one? { |v| v == "delete" }
+        next if black_list.include?(resource["kind"])
+        [resource["apigroup"], resource["version"], resource["kind"]].compact.join("/")
       end.compact
     end
 
-    # kubectl api-resources -o wide returns 5 columns
-    # NAME SHORTNAMES APIGROUP NAMESPACED KIND VERBS
-    # SHORTNAMES and APIGROUP may be blank
-    # VERBS is an array
-    # serviceaccounts sa <blank> true ServiceAccount [create delete deletecollection get list patch update watch]
     def fetch_resources(namespaced: false)
-      command = %w(api-resources)
-      command << "--namespaced=#{namespaced}"
-      raw, err, st = kubectl.run(*command, output: "wide", attempts: 5,
-        use_namespace: false)
-      if st.success?
-        rows = raw.split("\n")
-        header = rows[0]
-        resources = rows[1..-1]
-        full_width_field_names = header.downcase.scan(/[a-z]+[\W]*/)
-        cursor = 0
-        fields = full_width_field_names.each_with_object({}) do |name, hash|
-          start = cursor
-          cursor = start + name.length
-          # Last field should consume the remainder of the line
-          cursor = 0 if full_width_field_names.last == name.strip
-          hash[name.strip] = [start, cursor - 1]
-        end
-        resources.map do |resource|
-          resource = fields.map { |k, (s, e)| [k.strip, resource[s..e].strip] }.to_h
-          # Manually parse verbs: "[get list]" into %w(get list)
-          resource["verbs"] = resource["verbs"][1..-2].split
-          resource
-        end
-      else
-        raise FatalKubeAPIError, "Error retrieving api-resources: #{err}"
+      responses = Concurrent::Hash.new
+      Krane::Concurrency.split_across_threads(api_paths) do |path|
+        responses[path] = fetch_api_path(path)["resources"] || []
       end
+      responses.flat_map do |path, resources|
+        resources.map { |r| resource_hash(path, namespaced, r) }
+      end.compact.uniq { |r| r["kind"] }
     end
 
-    private
-
-    # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
-    # A kind may not exist in all versions of the group.
-    def fetch_api_versions
-      raw, err, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
-      # The "core" group is represented by an empty string
-      versions = { "" => %w(v1) }
+    def fetch_mutating_webhook_configurations
+      command = %w(get mutatingwebhookconfigurations)
+      raw_json, err, st = kubectl.run(*command, output: "json", attempts: 5, use_namespace: false)
       if st.success?
-        rows = raw.split("\n")
-        rows.each do |group_version|
-          group, version = group_version.split("/")
-          versions[group] ||= []
-          versions[group] << version
+        JSON.parse(raw_json)["items"].map do |definition|
+          Krane::MutatingWebhookConfiguration.new(namespace: namespace, context: context, logger: logger,
+            definition: definition, statsd_tags: @namespace_tags)
         end
       else
-        raise FatalKubeAPIError, "Error retrieving api-versions: #{err}"
+        raise FatalKubeAPIError, "Error retrieving mutatingwebhookconfigurations: #{err}"
       end
-      versions
     end
 
-    def version_for_kind(versions, kind)
-      # Override list for kinds that don't appear in the lastest version of a group
-      version_override = { "CronJob" => "v1beta1", "VolumeAttachment" => "v1beta1",
-                           "CSIDriver" => "v1beta1", "Ingress" => "v1beta1",
-                           "CSINode" => "v1beta1", "Job" => "v1",
-                           "IngressClass" => "v1beta1", "FrontendConfig" => "v1beta1",
-                           "ServiceNetworkEndpointGroup" => "v1beta1",
-                           "EnvoyFilter" => "v1alpha3",
-                           "TCPIngress" => "v1beta1" }
+    private
 
-      pattern = /v(?<major>\d+)(?<pre>alpha|beta)?(?<minor>\d+)?/
-      latest = versions.sort_by do |version|
-        match = version.match(pattern)
-        pre = { "alpha" => 0, "beta" => 1, nil => 2 }.fetch(match[:pre])
-        [match[:major].to_i, pre, match[:minor].to_i]
-      end.last
-      version_override.fetch(kind, latest)
+    def api_paths
+      @api_path_cache["/"] ||= begin
+        raw_json, err, st = kubectl.run("get", "--raw", "/", attempts: 5, use_namespace: false)
+        paths = if st.success?
+          JSON.parse(raw_json)["paths"]
+        else
+          raise FatalKubeAPIError, "Error retrieving raw path /: #{err}"
+        end
+        paths.select { |path| %r{^\/api.*\/v.*$}.match(path) }
+      end
     end
 
-    def gvk_string(api_versions, resource)
-      apiversion = resource['apiversion'].to_s
-
-      ## In kubectl 1.20 APIGroups was replaced by APIVersions
-      if apiversion.empty?
-        apigroup = resource['apigroup'].to_s
-        group_versions = api_versions[apigroup]
-
-        version = version_for_kind(group_versions, resource['kind'])
-        apigroup = 'core' if apigroup.empty?
-        apiversion = "#{apigroup}/#{version}"
+    def fetch_api_path(path)
+      @api_path_cache[path] ||= begin
+        raw_json, err, st = kubectl.run("get", "--raw", path, attempts: 2, use_namespace: false)
+        if st.success?
+          JSON.parse(raw_json)
+        else
+          logger.warn("Error retrieving api path: #{err}")
+          {}
+        end
       end
+    end
 
-      apiversion = "core/#{apiversion}" unless apiversion.include?("/")
-      [apiversion, resource['kind']].compact.join("/")
+    def resource_hash(path, namespaced, blob)
+      return unless blob["namespaced"] == namespaced
+      # skip sub-resources
+      return if blob["name"].include?("/")
+      path_regex = %r{(/apis?/)(?<group>[^/]*)/?(?<version>v.+)}
+      match = path.match(path_regex)
+      {
+        "verbs" => blob["verbs"],
+        "kind" => blob["kind"],
+        "apigroup" => match[:group],
+        "version" => match[:version],
+      }
     end
 
     def fetch_crds

data/lib/krane/container_logs.rb

@@ -3,7 +3,7 @@ module Krane
   class ContainerLogs
     attr_reader :lines, :container_name
 
-    DEFAULT_LINE_LIMIT = 25
+    DEFAULT_LINE_LIMIT = Integer(ENV.fetch('KRANE_LOG_LINE_LIMIT', 25))
 
     def initialize(parent_id:, container_name:, namespace:, context:, logger:)
       @parent_id = parent_id

data/lib/krane/deploy_task.rb

@@ -30,6 +30,7 @@ require 'krane/kubernetes_resource'
   custom_resource_definition
   horizontal_pod_autoscaler
   secret
+  mutating_webhook_configuration
 ).each do |subresource|
   require "krane/kubernetes_resource/#{subresource}"
 end
@@ -277,16 +278,25 @@ module Krane
     end
     measure_method(:validate_configuration)
 
+    def partition_dry_run_resources(resources)
+      individuals = []
+      mutating_webhook_configurations = cluster_resource_discoverer.fetch_mutating_webhook_configurations
+      mutating_webhook_configurations.each do |mutating_webhook_configuration|
+        mutating_webhook_configuration.webhooks.each do |webhook|
+          individuals = (individuals + resources.select { |resource| webhook.matches_resource?(resource) }).uniq
+          resources -= individuals
+        end
+      end
+      [resources, individuals]
+    end
+
     def validate_resources(resources)
       validate_globals(resources)
-      batch_dry_run_success = validate_dry_run(resources)
+      batchable_resources, individuals = partition_dry_run_resources(resources.dup)
+      batch_dry_run_success = kubectl.server_dry_run_enabled? && validate_dry_run(batchable_resources)
+      individuals += batchable_resources unless batch_dry_run_success
       Krane::Concurrency.split_across_threads(resources) do |r|
-        # No need to pass in kubectl (and do per-resource dry run apply) if batch dry run succeeded
-        if batch_dry_run_success
-          r.validate_definition(kubectl: nil, selector: @selector, dry_run: false)
-        else
-          r.validate_definition(kubectl: kubectl, selector: @selector, dry_run: true)
-        end
+        r.validate_definition(kubectl: kubectl, selector: @selector, dry_run: individuals.include?(r))
       end
       failed_resources = resources.select(&:validation_failed?)
       if failed_resources.present?

data/lib/krane/kubectl.rb

@@ -101,7 +101,7 @@ module Krane
     end
 
     def dry_run_flag
-      if server_version >= Gem::Version.new("1.18")
+      if client_version >= Gem::Version.new("1.18")
         "--dry-run=server"
       else
         "--server-dry-run"

data/lib/krane/kubernetes_resource.rb

@@ -226,6 +226,11 @@ module Krane
       version ? grouping : "core"
     end
 
+    def version
+      prefix, version = @definition.dig("apiVersion").split("/")
+      version || prefix
+    end
+
     def kubectl_resource_type
       type
     end
@@ -560,7 +565,7 @@ module Krane
 
     # Server side dry run is only supported on apply
     def validate_with_server_side_dry_run(kubectl)
-      command = if kubectl.server_version >= Gem::Version.new('1.18')
+      command = if kubectl.client_version >= Gem::Version.new('1.18')
         ["apply", "-f", file_path, "--dry-run=server", "--output=name"]
       else
         ["apply", "-f", file_path, "--server-dry-run", "--output=name"]

data/lib/krane/kubernetes_resource/mutating_webhook_configuration.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Krane
+  class MutatingWebhookConfiguration < KubernetesResource
+    GLOBAL = true
+
+    class Webhook
+      EQUIVALENT = 'Equivalent'
+      EXACT = 'Exact'
+
+      class Rule
+        def initialize(definition)
+          @definition = definition
+        end
+
+        def matches_resource?(resource, accept_equivalent:)
+          groups.each do |group|
+            versions.each do |version|
+              resources.each do |kind|
+                return true if (resource.group == group || group == '*' || accept_equivalent) &&
+                  (resource.version == version || version == '*' || accept_equivalent) &&
+                  (resource.type.downcase == kind.downcase.singularize || kind == "*")
+              end
+            end
+          end
+          false
+        end
+
+        def groups
+          @definition.dig('apiGroups')
+        end
+
+        def versions
+          @definition.dig('apiVersions')
+        end
+
+        def resources
+          @definition.dig('resources')
+        end
+      end
+
+      def initialize(definition)
+        @definition = definition
+      end
+
+      def side_effects
+        @definition.dig('sideEffects')
+      end
+
+      def has_side_effects?
+        !%w(None NoneOnDryRun).include?(side_effects)
+      end
+
+      def match_policy
+        @definition.dig('matchPolicy')
+      end
+
+      def matches_resource?(resource, skip_rule_if_side_effect_none: true)
+        return false if skip_rule_if_side_effect_none && !has_side_effects?
+        rules.any? do |rule|
+          rule.matches_resource?(resource, accept_equivalent: match_policy == EQUIVALENT)
+        end
+      end
+
+      def rules
+        @definition.fetch('rules', []).map { |rule| Rule.new(rule) }
+      end
+    end
+
+    def initialize(namespace:, context:, definition:, logger:, statsd_tags:)
+      @webhooks = (definition.dig('webhooks') || []).map { |hook| Webhook.new(hook) }
+      super(namespace: namespace, context: context, definition: definition,
+        logger: logger, statsd_tags: statsd_tags)
+    end
+
+    TIMEOUT = 30.seconds
+
+    def deploy_succeeded?
+      exists?
+    end
+
+    def webhooks
+      @definition.fetch('webhooks', []).map { |webhook| Webhook.new(webhook) }
+    end
+  end
+end

data/lib/krane/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Krane
-  VERSION = "2.1.4"
+  VERSION = "2.1.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krane
 version: !ruby/object:Gem::Version
-  version: 2.1.4
+  version: 2.1.9
 platform: ruby
 authors:
 - Katrina Verey
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-01-20 00:00:00.000000000 Z
+date: 2021-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -91,7 +91,7 @@ dependencies:
         version: '2.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -101,7 +101,7 @@ dependencies:
         version: '2.8'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -374,6 +374,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.89.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-shopify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -460,6 +474,7 @@ files:
 - lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb
 - lib/krane/kubernetes_resource/ingress.rb
 - lib/krane/kubernetes_resource/job.rb
+- lib/krane/kubernetes_resource/mutating_webhook_configuration.rb
 - lib/krane/kubernetes_resource/network_policy.rb
 - lib/krane/kubernetes_resource/persistent_volume_claim.rb
 - lib/krane/kubernetes_resource/pod.rb
@@ -493,11 +508,6 @@ files:
 - lib/krane/task_config_validator.rb
 - lib/krane/template_sets.rb
 - lib/krane/version.rb
-- screenshots/deploy-demo.gif
-- screenshots/migrate-logs.png
-- screenshots/missing-secret-fail.png
-- screenshots/success.png
-- screenshots/test-output.png
 homepage: https://github.com/Shopify/krane
 licenses:
 - MIT
@@ -518,7 +528,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: A command line tool that helps you ship changes to a Kubernetes namespace