krane 2.1.5 → 2.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ae99732594c24143e48be0c4c44fdfec9f3f81e7d604362f8d000df837758aaa
4
- data.tar.gz: 11684058ac036a9846e5101944c0f26e33216da3d87bde0f20fe13246e52bbb5
3
+ metadata.gz: d496112ecd7ff05ab81ab6cad622632a7d169b38d1ad0340c5dc2088aa061677
4
+ data.tar.gz: 16c47fa820e244a7d42080e3fdf81065eda9167dda898e12412ab5c72d5bd4be
5
5
  SHA512:
6
- metadata.gz: 36fec016df1d46e0bec3b7889913bbaa20bdb411795bb58b9995a87dde2bd0ed94203b21d229673fa8fe3d663d60981ce0e4d666acc173e7262b67a3671209ac
7
- data.tar.gz: df91b23d97e65489476c5ddb9f404e07c7a314003d84debd6cb5d59c27328f9a48394876af32a7ccb3976d3bfb296f60cd54af4c4fbdca6c31527ce86c2ad19a
6
+ metadata.gz: 97eeceba4a6f132ec19bd40bfdcf9fb7ad94f0a791f784d6b3ec0e01eb78eddca2e6d5e6809fc2feb15d6f5a4aba9c7eee295e4660998d595d267700566d7ad5
7
+ data.tar.gz: 1c364bd503c83eb902e548d68d06aed5235e1ef33c8e72e3fe2fd90871da76d8580be288b822ac768fb75a1dd7ed0c1c801634a0d7068a18d74b8263670b0f7c
data/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  ## next
2
2
 
3
+ *Enhancements*
4
+ - Remove the need for a hard coded GVK overide list via improvements to cluster discovery [#778](https://github.com/Shopify/krane/pull/778)
5
+
6
+ *Bug Fixes*
7
+ - Remove resources that are targeted by side-effect-inducing mutating admission webhooks from the serverside dry run batch [#798](https://github.com/Shopify/krane/pull/798)
8
+
3
9
  ## 2.1.5
4
10
 
5
11
  - Fix bug where the wrong dry-run flag is used for kubectl if client version is below 1.18 AND server version is 1.18+ [#793](https://github.com/Shopify/krane/pull/793).
@@ -1,4 +1,5 @@
1
1
  # frozen_string_literal: true
2
+ require 'concurrent'
2
3
 
3
4
  module Krane
4
5
  class ClusterResourceDiscovery
@@ -7,6 +8,7 @@ module Krane
7
8
  def initialize(task_config:, namespace_tags: [])
8
9
  @task_config = task_config
9
10
  @namespace_tags = namespace_tags
11
+ @api_path_cache = {}
10
12
  end
11
13
 
12
14
  def crds
@@ -18,104 +20,74 @@ module Krane
18
20
 
19
21
  def prunable_resources(namespaced:)
20
22
  black_list = %w(Namespace Node ControllerRevision)
21
- api_versions = fetch_api_versions
22
-
23
- fetch_resources(namespaced: namespaced).uniq { |r| r['kind'] }.map do |resource|
24
- next unless resource['verbs'].one? { |v| v == "delete" }
25
- next if black_list.include?(resource['kind'])
26
- gvk_string(api_versions, resource)
23
+ fetch_resources(namespaced: namespaced).map do |resource|
24
+ next unless resource["verbs"].one? { |v| v == "delete" }
25
+ next if black_list.include?(resource["kind"])
26
+ [resource["apigroup"], resource["version"], resource["kind"]].compact.join("/")
27
27
  end.compact
28
28
  end
29
29
 
30
- # kubectl api-resources -o wide returns 5 columns
31
- # NAME SHORTNAMES APIGROUP NAMESPACED KIND VERBS
32
- # SHORTNAMES and APIGROUP may be blank
33
- # VERBS is an array
34
- # serviceaccounts sa <blank> true ServiceAccount [create delete deletecollection get list patch update watch]
35
30
  def fetch_resources(namespaced: false)
36
- command = %w(api-resources)
37
- command << "--namespaced=#{namespaced}"
38
- raw, err, st = kubectl.run(*command, output: "wide", attempts: 5,
39
- use_namespace: false)
40
- if st.success?
41
- rows = raw.split("\n")
42
- header = rows[0]
43
- resources = rows[1..-1]
44
- full_width_field_names = header.downcase.scan(/[a-z]+[\W]*/)
45
- cursor = 0
46
- fields = full_width_field_names.each_with_object({}) do |name, hash|
47
- start = cursor
48
- cursor = start + name.length
49
- # Last field should consume the remainder of the line
50
- cursor = 0 if full_width_field_names.last == name.strip
51
- hash[name.strip] = [start, cursor - 1]
52
- end
53
- resources.map do |resource|
54
- resource = fields.map { |k, (s, e)| [k.strip, resource[s..e].strip] }.to_h
55
- # Manually parse verbs: "[get list]" into %w(get list)
56
- resource["verbs"] = resource["verbs"][1..-2].split
57
- resource
58
- end
59
- else
60
- raise FatalKubeAPIError, "Error retrieving api-resources: #{err}"
31
+ responses = Concurrent::Hash.new
32
+ Krane::Concurrency.split_across_threads(api_paths) do |path|
33
+ responses[path] = fetch_api_path(path)["resources"] || []
61
34
  end
35
+ responses.flat_map do |path, resources|
36
+ resources.map { |r| resource_hash(path, namespaced, r) }
37
+ end.compact.uniq { |r| r["kind"] }
62
38
  end
63
39
 
64
- private
65
-
66
- # kubectl api-versions returns a list of group/version strings e.g. autoscaling/v2beta2
67
- # A kind may not exist in all versions of the group.
68
- def fetch_api_versions
69
- raw, err, st = kubectl.run("api-versions", attempts: 5, use_namespace: false)
70
- # The "core" group is represented by an empty string
71
- versions = { "" => %w(v1) }
40
+ def fetch_mutating_webhook_configurations
41
+ command = %w(get mutatingwebhookconfigurations)
42
+ raw_json, err, st = kubectl.run(*command, output: "json", attempts: 5, use_namespace: false)
72
43
  if st.success?
73
- rows = raw.split("\n")
74
- rows.each do |group_version|
75
- group, version = group_version.split("/")
76
- versions[group] ||= []
77
- versions[group] << version
44
+ JSON.parse(raw_json)["items"].map do |definition|
45
+ Krane::MutatingWebhookConfiguration.new(namespace: namespace, context: context, logger: logger,
46
+ definition: definition, statsd_tags: @namespace_tags)
78
47
  end
79
48
  else
80
- raise FatalKubeAPIError, "Error retrieving api-versions: #{err}"
49
+ raise FatalKubeAPIError, "Error retrieving mutatingwebhookconfigurations: #{err}"
81
50
  end
82
- versions
83
51
  end
84
52
 
85
- def version_for_kind(versions, kind)
86
- # Override list for kinds that don't appear in the lastest version of a group
87
- version_override = { "CronJob" => "v1beta1", "VolumeAttachment" => "v1beta1",
88
- "CSIDriver" => "v1beta1", "Ingress" => "v1beta1",
89
- "CSINode" => "v1beta1", "Job" => "v1",
90
- "IngressClass" => "v1beta1", "FrontendConfig" => "v1beta1",
91
- "ServiceNetworkEndpointGroup" => "v1beta1",
92
- "EnvoyFilter" => "v1alpha3",
93
- "TCPIngress" => "v1beta1" }
53
+ private
94
54
 
95
- pattern = /v(?<major>\d+)(?<pre>alpha|beta)?(?<minor>\d+)?/
96
- latest = versions.sort_by do |version|
97
- match = version.match(pattern)
98
- pre = { "alpha" => 0, "beta" => 1, nil => 2 }.fetch(match[:pre])
99
- [match[:major].to_i, pre, match[:minor].to_i]
100
- end.last
101
- version_override.fetch(kind, latest)
55
+ def api_paths
56
+ @api_path_cache["/"] ||= begin
57
+ raw_json, err, st = kubectl.run("get", "--raw", "/", attempts: 5, use_namespace: false)
58
+ paths = if st.success?
59
+ JSON.parse(raw_json)["paths"]
60
+ else
61
+ raise FatalKubeAPIError, "Error retrieving raw path /: #{err}"
62
+ end
63
+ paths.select { |path| %r{^\/api.*\/v.*$}.match(path) }
64
+ end
102
65
  end
103
66
 
104
- def gvk_string(api_versions, resource)
105
- apiversion = resource['apiversion'].to_s
106
-
107
- ## In kubectl 1.20 APIGroups was replaced by APIVersions
108
- if apiversion.empty?
109
- apigroup = resource['apigroup'].to_s
110
- group_versions = api_versions[apigroup]
111
-
112
- version = version_for_kind(group_versions, resource['kind'])
113
- apigroup = 'core' if apigroup.empty?
114
- apiversion = "#{apigroup}/#{version}"
67
+ def fetch_api_path(path)
68
+ @api_path_cache[path] ||= begin
69
+ raw_json, err, st = kubectl.run("get", "--raw", path, attempts: 2, use_namespace: false)
70
+ if st.success?
71
+ JSON.parse(raw_json)
72
+ else
73
+ logger.warn("Error retrieving api path: #{err}")
74
+ {}
75
+ end
115
76
  end
77
+ end
116
78
 
117
- apiversion = "core/#{apiversion}" unless apiversion.include?("/")
118
- [apiversion, resource['kind']].compact.join("/")
79
+ def resource_hash(path, namespaced, blob)
80
+ return unless blob["namespaced"] == namespaced
81
+ # skip sub-resources
82
+ return if blob["name"].include?("/")
83
+ path_regex = %r{(/apis?/)(?<group>[^/]*)/?(?<version>v.+)}
84
+ match = path.match(path_regex)
85
+ {
86
+ "verbs" => blob["verbs"],
87
+ "kind" => blob["kind"],
88
+ "apigroup" => match[:group],
89
+ "version" => match[:version],
90
+ }
119
91
  end
120
92
 
121
93
  def fetch_crds
@@ -129,7 +101,7 @@ module Krane
129
101
  end
130
102
 
131
103
  def kubectl
132
- @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
104
+ @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true, default_timeout: 1)
133
105
  end
134
106
  end
135
107
  end
@@ -30,6 +30,7 @@ require 'krane/kubernetes_resource'
30
30
  custom_resource_definition
31
31
  horizontal_pod_autoscaler
32
32
  secret
33
+ mutating_webhook_configuration
33
34
  ).each do |subresource|
34
35
  require "krane/kubernetes_resource/#{subresource}"
35
36
  end
@@ -277,16 +278,25 @@ module Krane
277
278
  end
278
279
  measure_method(:validate_configuration)
279
280
 
281
+ def partition_dry_run_resources(resources)
282
+ individuals = []
283
+ mutating_webhook_configurations = cluster_resource_discoverer.fetch_mutating_webhook_configurations
284
+ mutating_webhook_configurations.each do |mutating_webhook_configuration|
285
+ mutating_webhook_configuration.webhooks.each do |webhook|
286
+ individuals = (individuals + resources.select { |resource| webhook.matches_resource?(resource) }).uniq
287
+ resources -= individuals
288
+ end
289
+ end
290
+ [resources, individuals]
291
+ end
292
+
280
293
  def validate_resources(resources)
281
294
  validate_globals(resources)
282
- batch_dry_run_success = kubectl.server_dry_run_enabled? && validate_dry_run(resources)
295
+ batchable_resources, individuals = partition_dry_run_resources(resources.dup)
296
+ batch_dry_run_success = kubectl.server_dry_run_enabled? && validate_dry_run(batchable_resources)
297
+ individuals += batchable_resources unless batch_dry_run_success
283
298
  Krane::Concurrency.split_across_threads(resources) do |r|
284
- # No need to pass in kubectl (and do per-resource dry run apply) if batch dry run succeeded
285
- if batch_dry_run_success
286
- r.validate_definition(kubectl: nil, selector: @selector, dry_run: false)
287
- else
288
- r.validate_definition(kubectl: kubectl, selector: @selector, dry_run: true)
289
- end
299
+ r.validate_definition(kubectl: kubectl, selector: @selector, dry_run: individuals.include?(r))
290
300
  end
291
301
  failed_resources = resources.select(&:validation_failed?)
292
302
  if failed_resources.present?
@@ -226,6 +226,11 @@ module Krane
226
226
  version ? grouping : "core"
227
227
  end
228
228
 
229
+ def version
230
+ prefix, version = @definition.dig("apiVersion").split("/")
231
+ version || prefix
232
+ end
233
+
229
234
  def kubectl_resource_type
230
235
  type
231
236
  end
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Krane
4
+ class MutatingWebhookConfiguration < KubernetesResource
5
+ GLOBAL = true
6
+
7
+ class Webhook
8
+ EQUIVALENT = 'Equivalent'
9
+ EXACT = 'Exact'
10
+
11
+ class Rule
12
+ def initialize(definition)
13
+ @definition = definition
14
+ end
15
+
16
+ def matches_resource?(resource, accept_equivalent:)
17
+ groups.each do |group|
18
+ versions.each do |version|
19
+ resources.each do |kind|
20
+ return true if (resource.group == group || group == '*' || accept_equivalent) &&
21
+ (resource.version == version || version == '*' || accept_equivalent) &&
22
+ (resource.type.downcase == kind.downcase.singularize || kind == "*")
23
+ end
24
+ end
25
+ end
26
+ false
27
+ end
28
+
29
+ def groups
30
+ @definition.dig('apiGroups')
31
+ end
32
+
33
+ def versions
34
+ @definition.dig('apiVersions')
35
+ end
36
+
37
+ def resources
38
+ @definition.dig('resources')
39
+ end
40
+ end
41
+
42
+ def initialize(definition)
43
+ @definition = definition
44
+ end
45
+
46
+ def side_effects
47
+ @definition.dig('sideEffects')
48
+ end
49
+
50
+ def has_side_effects?
51
+ !%w(None NoneOnDryRun).include?(side_effects)
52
+ end
53
+
54
+ def match_policy
55
+ @definition.dig('matchPolicy')
56
+ end
57
+
58
+ def matches_resource?(resource, skip_rule_if_side_effect_none: true)
59
+ return false if skip_rule_if_side_effect_none && !has_side_effects?
60
+ rules.any? do |rule|
61
+ rule.matches_resource?(resource, accept_equivalent: match_policy == EQUIVALENT)
62
+ end
63
+ end
64
+
65
+ def rules
66
+ @definition.fetch('rules', []).map { |rule| Rule.new(rule) }
67
+ end
68
+ end
69
+
70
+ def initialize(namespace:, context:, definition:, logger:, statsd_tags:)
71
+ @webhooks = (definition.dig('webhooks') || []).map { |hook| Webhook.new(hook) }
72
+ super(namespace: namespace, context: context, definition: definition,
73
+ logger: logger, statsd_tags: statsd_tags)
74
+ end
75
+
76
+ TIMEOUT = 30.seconds
77
+
78
+ def deploy_succeeded?
79
+ exists?
80
+ end
81
+
82
+ def webhooks
83
+ @definition.fetch('webhooks', []).map { |webhook| Webhook.new(webhook) }
84
+ end
85
+ end
86
+ end
data/lib/krane/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
  module Krane
3
- VERSION = "2.1.5"
3
+ VERSION = "2.1.6"
4
4
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: krane
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.5
4
+ version: 2.1.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Katrina Verey
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2021-01-27 00:00:00.000000000 Z
13
+ date: 2021-02-11 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activesupport
@@ -460,6 +460,7 @@ files:
460
460
  - lib/krane/kubernetes_resource/horizontal_pod_autoscaler.rb
461
461
  - lib/krane/kubernetes_resource/ingress.rb
462
462
  - lib/krane/kubernetes_resource/job.rb
463
+ - lib/krane/kubernetes_resource/mutating_webhook_configuration.rb
463
464
  - lib/krane/kubernetes_resource/network_policy.rb
464
465
  - lib/krane/kubernetes_resource/persistent_volume_claim.rb
465
466
  - lib/krane/kubernetes_resource/pod.rb