kubernetes-deploy 0.29.0 → 1.0.0.pre.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.buildkite/pipeline.nightly.yml +7 -0
  3. data/.rubocop.yml +0 -12
  4. data/.shopify-build/{kubernetes-deploy.yml → krane.yml} +8 -2
  5. data/1.0-Upgrade.md +109 -0
  6. data/CHANGELOG.md +60 -0
  7. data/CONTRIBUTING.md +2 -2
  8. data/Gemfile +1 -0
  9. data/README.md +86 -2
  10. data/dev.yml +3 -1
  11. data/dev/flamegraph-from-tests +1 -1
  12. data/exe/kubernetes-deploy +12 -9
  13. data/exe/kubernetes-render +9 -7
  14. data/exe/kubernetes-restart +3 -3
  15. data/exe/kubernetes-run +1 -1
  16. data/kubernetes-deploy.gemspec +5 -5
  17. data/lib/krane.rb +5 -3
  18. data/lib/{kubernetes-deploy → krane}/bindings_parser.rb +1 -1
  19. data/lib/krane/cli/deploy_command.rb +25 -13
  20. data/lib/krane/cli/global_deploy_command.rb +55 -0
  21. data/lib/krane/cli/krane.rb +12 -3
  22. data/lib/krane/cli/render_command.rb +19 -9
  23. data/lib/krane/cli/restart_command.rb +4 -4
  24. data/lib/krane/cli/run_command.rb +4 -4
  25. data/lib/krane/cli/version_command.rb +1 -1
  26. data/lib/krane/cluster_resource_discovery.rb +113 -0
  27. data/lib/{kubernetes-deploy → krane}/common.rb +8 -9
  28. data/lib/krane/concerns/template_reporting.rb +29 -0
  29. data/lib/{kubernetes-deploy → krane}/concurrency.rb +1 -1
  30. data/lib/{kubernetes-deploy → krane}/container_logs.rb +3 -2
  31. data/lib/{kubernetes-deploy → krane}/deferred_summary_logging.rb +2 -2
  32. data/lib/{kubernetes-deploy → krane}/delayed_exceptions.rb +0 -0
  33. data/lib/krane/deploy_task.rb +16 -0
  34. data/lib/krane/deploy_task_config_validator.rb +29 -0
  35. data/lib/krane/deprecated_deploy_task.rb +404 -0
  36. data/lib/{kubernetes-deploy → krane}/duration_parser.rb +1 -3
  37. data/lib/{kubernetes-deploy → krane}/ejson_secret_provisioner.rb +10 -13
  38. data/lib/krane/errors.rb +28 -0
  39. data/lib/{kubernetes-deploy → krane}/formatted_logger.rb +2 -2
  40. data/lib/krane/global_deploy_task.rb +210 -0
  41. data/lib/krane/global_deploy_task_config_validator.rb +12 -0
  42. data/lib/{kubernetes-deploy → krane}/kubeclient_builder.rb +13 -5
  43. data/lib/{kubernetes-deploy → krane}/kubectl.rb +14 -16
  44. data/lib/{kubernetes-deploy → krane}/kubernetes_resource.rb +110 -27
  45. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/cloudsql.rb +1 -1
  46. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/config_map.rb +1 -1
  47. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/cron_job.rb +1 -1
  48. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/custom_resource.rb +2 -2
  49. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/custom_resource_definition.rb +1 -5
  50. data/lib/krane/kubernetes_resource/daemon_set.rb +90 -0
  51. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/deployment.rb +2 -2
  52. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/horizontal_pod_autoscaler.rb +1 -1
  53. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/ingress.rb +1 -1
  54. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/job.rb +1 -1
  55. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/network_policy.rb +1 -1
  56. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/persistent_volume_claim.rb +1 -1
  57. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod.rb +6 -2
  58. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod_disruption_budget.rb +2 -2
  59. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod_set_base.rb +3 -3
  60. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/pod_template.rb +1 -1
  61. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/replica_set.rb +2 -2
  62. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/resource_quota.rb +1 -1
  63. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/role.rb +1 -1
  64. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/role_binding.rb +1 -1
  65. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/secret.rb +1 -1
  66. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/service.rb +2 -2
  67. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/service_account.rb +1 -1
  68. data/lib/{kubernetes-deploy → krane}/kubernetes_resource/stateful_set.rb +2 -2
  69. data/lib/{kubernetes-deploy → krane}/label_selector.rb +1 -1
  70. data/lib/{kubernetes-deploy → krane}/oj.rb +0 -0
  71. data/lib/{kubernetes-deploy → krane}/options_helper.rb +2 -2
  72. data/lib/{kubernetes-deploy → krane}/remote_logs.rb +2 -2
  73. data/lib/krane/render_task.rb +149 -0
  74. data/lib/{kubernetes-deploy → krane}/renderer.rb +1 -1
  75. data/lib/{kubernetes-deploy → krane}/resource_cache.rb +10 -9
  76. data/lib/krane/resource_deployer.rb +265 -0
  77. data/lib/{kubernetes-deploy → krane}/resource_watcher.rb +24 -25
  78. data/lib/krane/restart_task.rb +228 -0
  79. data/lib/{kubernetes-deploy → krane}/rollout_conditions.rb +1 -1
  80. data/lib/krane/runner_task.rb +212 -0
  81. data/lib/{kubernetes-deploy → krane}/runner_task_config_validator.rb +1 -1
  82. data/lib/{kubernetes-deploy → krane}/statsd.rb +13 -27
  83. data/lib/krane/task_config.rb +22 -0
  84. data/lib/{kubernetes-deploy → krane}/task_config_validator.rb +1 -1
  85. data/lib/{kubernetes-deploy → krane}/template_sets.rb +5 -5
  86. data/lib/krane/version.rb +4 -0
  87. data/lib/kubernetes-deploy/deploy_task.rb +6 -608
  88. data/lib/kubernetes-deploy/errors.rb +1 -26
  89. data/lib/kubernetes-deploy/render_task.rb +5 -122
  90. data/lib/kubernetes-deploy/rescue_krane_exceptions.rb +18 -0
  91. data/lib/kubernetes-deploy/restart_task.rb +6 -198
  92. data/lib/kubernetes-deploy/runner_task.rb +6 -184
  93. metadata +96 -70
  94. data/lib/kubernetes-deploy/cluster_resource_discovery.rb +0 -34
  95. data/lib/kubernetes-deploy/kubernetes_resource/daemon_set.rb +0 -54
  96. data/lib/kubernetes-deploy/task_config.rb +0 -16
  97. data/lib/kubernetes-deploy/version.rb +0 -4
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class NetworkPolicy < KubernetesResource
4
4
  TIMEOUT = 30.seconds
5
5
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class PersistentVolumeClaim < KubernetesResource
4
4
  TIMEOUT = 5.minutes
5
5
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class Pod < KubernetesResource
4
4
  TIMEOUT = 10.minutes
5
5
 
@@ -101,6 +101,10 @@ module KubernetesDeploy
101
101
  exists? && !@stream_logs # don't print them a second time
102
102
  end
103
103
 
104
+ def node_name
105
+ @instance_data.dig('spec', 'nodeName')
106
+ end
107
+
104
108
  private
105
109
 
106
110
  def failed_schedule_reason
@@ -137,7 +141,7 @@ module KubernetesDeploy
137
141
  end
138
142
 
139
143
  def logs
140
- @logs ||= KubernetesDeploy::RemoteLogs.new(
144
+ @logs ||= Krane::RemoteLogs.new(
141
145
  logger: @logger,
142
146
  parent_id: id,
143
147
  container_names: @containers.map(&:name),
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class PodDisruptionBudget < KubernetesResource
4
4
  TIMEOUT = 10.seconds
5
5
 
@@ -13,7 +13,7 @@ module KubernetesDeploy
13
13
 
14
14
  def deploy_method
15
15
  # Required until https://github.com/kubernetes/kubernetes/issues/45398 changes
16
- :replace_force
16
+ uses_generate_name? ? :create : :replace_force
17
17
  end
18
18
 
19
19
  def timeout_message
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
- require 'kubernetes-deploy/kubernetes_resource/pod'
2
+ require 'krane/kubernetes_resource/pod'
3
3
 
4
- module KubernetesDeploy
4
+ module Krane
5
5
  class PodSetBase < KubernetesResource
6
6
  def failure_message
7
7
  pods.map(&:failure_message).compact.uniq.join("\n")
@@ -19,7 +19,7 @@ module KubernetesDeploy
19
19
  end
20
20
 
21
21
  def fetch_debug_logs
22
- logs = KubernetesDeploy::RemoteLogs.new(
22
+ logs = Krane::RemoteLogs.new(
23
23
  logger: @logger,
24
24
  parent_id: id,
25
25
  container_names: container_names,
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class PodTemplate < KubernetesResource
4
4
  def status
5
5
  exists? ? "Available" : "Not Found"
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
- require 'kubernetes-deploy/kubernetes_resource/pod_set_base'
2
+ require 'krane/kubernetes_resource/pod_set_base'
3
3
 
4
- module KubernetesDeploy
4
+ module Krane
5
5
  class ReplicaSet < PodSetBase
6
6
  TIMEOUT = 5.minutes
7
7
  attr_reader :pods
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class ResourceQuota < KubernetesResource
4
4
  TIMEOUT = 30.seconds
5
5
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class Role < KubernetesResource
4
4
  TIMEOUT = 30.seconds
5
5
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class RoleBinding < KubernetesResource
4
4
  TIMEOUT = 30.seconds
5
5
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class Secret < KubernetesResource
4
4
  TIMEOUT = 30.seconds
5
5
  SENSITIVE_TEMPLATE_CONTENT = true
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
- require 'kubernetes-deploy/kubernetes_resource/pod'
2
+ require 'krane/kubernetes_resource/pod'
3
3
 
4
- module KubernetesDeploy
4
+ module Krane
5
5
  class Service < KubernetesResource
6
6
  TIMEOUT = 7.minutes
7
7
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
- module KubernetesDeploy
2
+ module Krane
3
3
  class ServiceAccount < KubernetesResource
4
4
  TIMEOUT = 30.seconds
5
5
 
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
- require 'kubernetes-deploy/kubernetes_resource/pod_set_base'
3
- module KubernetesDeploy
2
+ require 'krane/kubernetes_resource/pod_set_base'
3
+ module Krane
4
4
  class StatefulSet < PodSetBase
5
5
  TIMEOUT = 10.minutes
6
6
  ONDELETE = 'OnDelete'
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module KubernetesDeploy
3
+ module Krane
4
4
  class LabelSelector
5
5
  def self.parse(string)
6
6
  selector = {}
File without changes
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- module KubernetesDeploy
3
+ module Krane
4
4
  module OptionsHelper
5
5
  class OptionsError < StandardError; end
6
6
 
@@ -19,7 +19,7 @@ module KubernetesDeploy
19
19
  end
20
20
 
21
21
  if template_paths.include?("-")
22
- Dir.mktmpdir("kubernetes-deploy") do |dir|
22
+ Dir.mktmpdir("krane") do |dir|
23
23
  template_dir_from_stdin(temp_dir: dir)
24
24
  validated_paths << dir
25
25
  yield validated_paths
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
- require 'kubernetes-deploy/container_logs'
2
+ require 'krane/container_logs'
3
3
 
4
- module KubernetesDeploy
4
+ module Krane
5
5
  class RemoteLogs
6
6
  attr_reader :container_logs
7
7
 
@@ -0,0 +1,149 @@
1
+ # frozen_string_literal: true
2
+ require 'tempfile'
3
+
4
+ require 'krane/common'
5
+ require 'krane/renderer'
6
+ require 'krane/template_sets'
7
+
8
+ module Krane
9
+ # Render templates
10
+ class RenderTask
11
+ # Initializes the render task
12
+ #
13
+ # @param logger [Object] Logger object (defaults to an instance of Krane::FormattedLogger)
14
+ # @param current_sha [String] The SHA of the commit
15
+ # @param template_dir [String] Path to a directory with templates to render (deprecated)
16
+ # @param template_paths [Array<String>] An array of template paths to render
17
+ # @param bindings [Hash] Bindings parsed by Krane::BindingsParser
18
+ def initialize(logger: nil, current_sha:, template_dir: nil, template_paths: [], bindings:)
19
+ @logger = logger || Krane::FormattedLogger.build
20
+ @template_dir = template_dir
21
+ @template_paths = template_paths.map { |path| File.expand_path(path) }
22
+ @bindings = bindings
23
+ @current_sha = current_sha
24
+ end
25
+
26
+ # Runs the task, returning a boolean representing success or failure
27
+ #
28
+ # @return [Boolean]
29
+ def run(*args)
30
+ run!(*args)
31
+ true
32
+ rescue Krane::FatalDeploymentError
33
+ false
34
+ end
35
+
36
+ # Runs the task, raising exceptions in case of issues
37
+ #
38
+ # @param stream [IO] Place to stream the output to
39
+ # @param only_filenames [Array<String>] List of filenames to render
40
+ #
41
+ # @return [nil]
42
+ def run!(stream, only_filenames = [])
43
+ @logger.reset
44
+ @logger.phase_heading("Initializing render task")
45
+
46
+ ts = TemplateSets.from_dirs_and_files(paths: template_sets_paths(only_filenames), logger: @logger)
47
+
48
+ validate_configuration(ts, only_filenames)
49
+ count = render_templates(stream, ts)
50
+
51
+ @logger.summary.add_action("Successfully rendered #{count} template(s)")
52
+ @logger.print_summary(:success)
53
+ rescue Krane::FatalDeploymentError
54
+ @logger.print_summary(:failure)
55
+ raise
56
+ end
57
+
58
+ private
59
+
60
+ def template_sets_paths(only_filenames)
61
+ if @template_paths.present?
62
+ # Validation will catch @template_paths & @template_dir being present
63
+ @template_paths
64
+ elsif only_filenames.blank?
65
+ [File.expand_path(@template_dir || '')]
66
+ else
67
+ absolute_template_dir = File.expand_path(@template_dir || '')
68
+ only_filenames.map do |name|
69
+ File.join(absolute_template_dir, name)
70
+ end
71
+ end
72
+ end
73
+
74
+ def render_templates(stream, template_sets)
75
+ @logger.phase_heading("Rendering template(s)")
76
+ count = 0
77
+ template_sets.with_resource_definitions_and_filename(render_erb: true,
78
+ current_sha: @current_sha, bindings: @bindings, raw: true) do |rendered_content, filename|
79
+ write_to_stream(rendered_content, filename, stream)
80
+ count += 1
81
+ end
82
+
83
+ count
84
+ rescue Krane::InvalidTemplateError => exception
85
+ log_invalid_template(exception)
86
+ raise
87
+ end
88
+
89
+ def write_to_stream(rendered_content, filename, stream)
90
+ file_basename = File.basename(filename)
91
+ @logger.info("Rendering #{file_basename}...")
92
+ implicit = []
93
+ YAML.parse_stream(rendered_content, "<rendered> #{filename}") { |d| implicit << d.implicit }
94
+ if rendered_content.present?
95
+ stream.puts "---\n" if implicit.first
96
+ stream.puts rendered_content
97
+ @logger.info("Rendered #{file_basename}")
98
+ else
99
+ @logger.warn("Rendered #{file_basename} successfully, but the result was blank")
100
+ end
101
+ rescue Psych::SyntaxError => exception
102
+ raise InvalidTemplateError.new("Template is not valid YAML. #{exception.message}", filename: filename)
103
+ end
104
+
105
+ def validate_configuration(template_sets, filenames)
106
+ @logger.info("Validating configuration")
107
+ errors = []
108
+ if @template_dir.present? && @template_paths.present?
109
+ errors << "template_dir and template_paths can not be combined"
110
+ elsif @template_dir.blank? && @template_paths.blank?
111
+ errors << "template_dir or template_paths must be set"
112
+ end
113
+
114
+ if filenames.present?
115
+ if @template_dir.nil?
116
+ errors << "template_dir must be set to use filenames"
117
+ else
118
+ absolute_template_dir = File.expand_path(@template_dir)
119
+ filenames.each do |filename|
120
+ absolute_file = File.expand_path(File.join(@template_dir, filename))
121
+ unless absolute_file.start_with?(absolute_template_dir)
122
+ errors << "Filename \"#{absolute_file}\" is outside the template directory," \
123
+ " which was resolved as #{absolute_template_dir}"
124
+ end
125
+ end
126
+ end
127
+ end
128
+
129
+ errors += template_sets.validate
130
+
131
+ unless errors.empty?
132
+ @logger.summary.add_action("Configuration invalid")
133
+ @logger.summary.add_paragraph(errors.map { |err| "- #{err}" }.join("\n"))
134
+ raise Krane::TaskConfigurationError, "Configuration invalid: #{errors.join(', ')}"
135
+ end
136
+ end
137
+
138
+ def log_invalid_template(exception)
139
+ @logger.error("Failed to render #{exception.filename}")
140
+
141
+ debug_msg = ColorizedString.new("Invalid template: #{exception.filename}\n").red
142
+ debug_msg += "> Error message:\n#{FormattedLogger.indent_four(exception.to_s)}"
143
+ if exception.content
144
+ debug_msg += "\n> Template content:\n#{FormattedLogger.indent_four(exception.content)}"
145
+ end
146
+ @logger.summary.add_paragraph(debug_msg)
147
+ end
148
+ end
149
+ end
@@ -5,7 +5,7 @@ require 'securerandom'
5
5
  require 'yaml'
6
6
  require 'json'
7
7
 
8
- module KubernetesDeploy
8
+ module Krane
9
9
  class Renderer
10
10
  class InvalidPartialError < InvalidTemplateError
11
11
  attr_accessor :parents, :content, :filename
@@ -2,22 +2,22 @@
2
2
 
3
3
  require 'concurrent/hash'
4
4
 
5
- module KubernetesDeploy
5
+ module Krane
6
6
  class ResourceCache
7
- def initialize(namespace, context, logger)
8
- @namespace = namespace
9
- @context = context
10
- @logger = logger
7
+ delegate :namespace, :context, :logger, to: :@task_config
8
+
9
+ def initialize(task_config)
10
+ @task_config = task_config
11
11
 
12
12
  @kind_fetcher_locks = Concurrent::Hash.new { |hash, key| hash[key] = Mutex.new }
13
13
  @data = Concurrent::Hash.new
14
- @kubectl = Kubectl.new(namespace: @namespace, context: @context, logger: @logger, log_failure_by_default: false)
14
+ @kubectl = Kubectl.new(task_config: @task_config, log_failure_by_default: false)
15
15
  end
16
16
 
17
17
  def get_instance(kind, resource_name, raise_if_not_found: false)
18
18
  instance = use_or_populate_cache(kind).fetch(resource_name, {})
19
19
  if instance.blank? && raise_if_not_found
20
- raise KubernetesDeploy::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for kind #{kind})"
20
+ raise Krane::Kubectl::ResourceNotFoundError, "Resource does not exist (used cache for kind #{kind})"
21
21
  end
22
22
  instance
23
23
  rescue KubectlError
@@ -39,7 +39,7 @@ module KubernetesDeploy
39
39
  private
40
40
 
41
41
  def statsd_tags
42
- { namespace: @namespace, context: @context }
42
+ { namespace: namespace, context: context }
43
43
  end
44
44
 
45
45
  def use_or_populate_cache(kind)
@@ -51,9 +51,10 @@ module KubernetesDeploy
51
51
 
52
52
  def fetch_by_kind(kind)
53
53
  resource_class = KubernetesResource.class_for_kind(kind)
54
+ global_kind = @task_config.global_kinds.map(&:downcase).include?(kind.downcase)
54
55
  output_is_sensitive = resource_class.nil? ? false : resource_class::SENSITIVE_TEMPLATE_CONTENT
55
56
  raw_json, _, st = @kubectl.run("get", kind, "--chunk-size=0", attempts: 5, output: "json",
56
- output_is_sensitive: output_is_sensitive)
57
+ output_is_sensitive: output_is_sensitive, use_namespace: !global_kind)
57
58
  raise KubectlError unless st.success?
58
59
 
59
60
  instances = {}
@@ -0,0 +1,265 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'krane/resource_watcher'
4
+ require 'krane/concerns/template_reporting'
5
+
6
+ module Krane
7
+ class ResourceDeployer
8
+ extend Krane::StatsD::MeasureMethods
9
+ include Krane::TemplateReporting
10
+
11
+ delegate :logger, to: :@task_config
12
+ attr_reader :statsd_tags
13
+
14
+ def initialize(task_config:, prune_whitelist:, max_watch_seconds:, current_sha: nil, selector:, statsd_tags:)
15
+ @task_config = task_config
16
+ @prune_whitelist = prune_whitelist
17
+ @max_watch_seconds = max_watch_seconds
18
+ @current_sha = current_sha
19
+ @selector = selector
20
+ @statsd_tags = statsd_tags
21
+ end
22
+
23
+ def deploy!(resources, verify_result, prune)
24
+ if verify_result
25
+ deploy_all_resources(resources, prune: prune, verify: true)
26
+ failed_resources = resources.reject(&:deploy_succeeded?)
27
+ success = failed_resources.empty?
28
+ if !success && failed_resources.all?(&:deploy_timed_out?)
29
+ raise DeploymentTimeoutError
30
+ end
31
+ raise FatalDeploymentError unless success
32
+ else
33
+ deploy_all_resources(resources, prune: prune, verify: false)
34
+ logger.summary.add_action("deployed #{resources.length} #{'resource'.pluralize(resources.length)}")
35
+ warning = <<~MSG
36
+ Deploy result verification is disabled for this deploy.
37
+ This means the desired changes were communicated to Kubernetes, but the deploy did not make sure they actually succeeded.
38
+ MSG
39
+ logger.summary.add_paragraph(ColorizedString.new(warning).yellow)
40
+ end
41
+ end
42
+
43
+ def predeploy_priority_resources(resource_list, predeploy_sequence)
44
+ bare_pods = resource_list.select { |resource| resource.is_a?(Pod) }
45
+ if bare_pods.count == 1
46
+ bare_pods.first.stream_logs = true
47
+ end
48
+
49
+ predeploy_sequence.each do |resource_type|
50
+ matching_resources = resource_list.select { |r| r.type == resource_type }
51
+ next if matching_resources.empty?
52
+ deploy_resources(matching_resources, verify: true, record_summary: false)
53
+
54
+ failed_resources = matching_resources.reject(&:deploy_succeeded?)
55
+ fail_count = failed_resources.length
56
+ if fail_count > 0
57
+ Krane::Concurrency.split_across_threads(failed_resources) do |r|
58
+ r.sync_debug_info(kubectl)
59
+ end
60
+ failed_resources.each { |r| logger.summary.add_paragraph(r.debug_message) }
61
+ raise FatalDeploymentError, "Failed to deploy #{fail_count} priority #{'resource'.pluralize(fail_count)}"
62
+ end
63
+ logger.blank_line
64
+ end
65
+ end
66
+ measure_method(:predeploy_priority_resources, 'priority_resources.duration')
67
+
68
+ private
69
+
70
+ def deploy_all_resources(resources, prune: false, verify:, record_summary: true)
71
+ deploy_resources(resources, prune: prune, verify: verify, record_summary: record_summary)
72
+ end
73
+ measure_method(:deploy_all_resources, 'normal_resources.duration')
74
+
75
+ def deploy_resources(resources, prune: false, verify:, record_summary: true)
76
+ return if resources.empty?
77
+ deploy_started_at = Time.now.utc
78
+
79
+ if resources.length > 1
80
+ logger.info("Deploying resources:")
81
+ resources.each do |r|
82
+ logger.info("- #{r.id} (#{r.pretty_timeout_type})")
83
+ end
84
+ else
85
+ resource = resources.first
86
+ logger.info("Deploying #{resource.id} (#{resource.pretty_timeout_type})")
87
+ end
88
+
89
+ # Apply can be done in one large batch, the rest have to be done individually
90
+ applyables, individuals = resources.partition { |r| r.deploy_method == :apply }
91
+ # Prunable resources should also applied so that they can be pruned
92
+ pruneable_types = @prune_whitelist.map { |t| t.split("/").last }
93
+ applyables += individuals.select { |r| pruneable_types.include?(r.type) }
94
+
95
+ individuals.each do |individual_resource|
96
+ individual_resource.deploy_started_at = Time.now.utc
97
+
98
+ case individual_resource.deploy_method
99
+ when :create
100
+ err, status = create_resource(individual_resource)
101
+ when :replace
102
+ err, status = replace_or_create_resource(individual_resource)
103
+ when :replace_force
104
+ err, status = replace_or_create_resource(individual_resource, force: true)
105
+ else
106
+ # Fail Fast! This is a programmer mistake.
107
+ raise ArgumentError, "Unexpected deploy method! (#{individual_resource.deploy_method.inspect})"
108
+ end
109
+
110
+ next if status.success?
111
+
112
+ raise FatalDeploymentError, <<~MSG
113
+ Failed to replace or create resource: #{individual_resource.id}
114
+ #{individual_resource.sensitive_template_content? ? '<suppressed sensitive output>' : err}
115
+ MSG
116
+ end
117
+
118
+ apply_all(applyables, prune)
119
+
120
+ if verify
121
+ watcher = Krane::ResourceWatcher.new(resources: resources, deploy_started_at: deploy_started_at,
122
+ timeout: @max_watch_seconds, task_config: @task_config, sha: @current_sha)
123
+ watcher.run(record_summary: record_summary)
124
+ end
125
+ end
126
+
127
+ def apply_all(resources, prune)
128
+ return unless resources.present?
129
+ command = %w(apply)
130
+
131
+ Dir.mktmpdir do |tmp_dir|
132
+ resources.each do |r|
133
+ FileUtils.symlink(r.file_path, tmp_dir)
134
+ r.deploy_started_at = Time.now.utc
135
+ end
136
+ command.push("-f", tmp_dir)
137
+
138
+ if prune && @prune_whitelist.present?
139
+ command.push("--prune")
140
+ if @selector
141
+ command.push("--selector", @selector.to_s)
142
+ else
143
+ command.push("--all")
144
+ end
145
+ @prune_whitelist.each { |type| command.push("--prune-whitelist=#{type}") }
146
+ end
147
+
148
+ output_is_sensitive = resources.any?(&:sensitive_template_content?)
149
+ global_mode = resources.all?(&:global?)
150
+ out, err, st = kubectl.run(*command, log_failure: false, output_is_sensitive: output_is_sensitive,
151
+ use_namespace: !global_mode)
152
+
153
+ if st.success?
154
+ log_pruning(out) if prune
155
+ else
156
+ record_apply_failure(err, resources: resources)
157
+ raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
158
+ end
159
+ end
160
+ end
161
+ measure_method(:apply_all)
162
+
163
+ def log_pruning(kubectl_output)
164
+ pruned = kubectl_output.scan(/^(.*) pruned$/)
165
+ return unless pruned.present?
166
+
167
+ logger.info("The following resources were pruned: #{pruned.join(', ')}")
168
+ logger.summary.add_action("pruned #{pruned.length} #{'resource'.pluralize(pruned.length)}")
169
+ end
170
+
171
+ def record_apply_failure(err, resources: [])
172
+ warn_msg = "WARNING: Any resources not mentioned in the error(s) below were likely created/updated. " \
173
+ "You may wish to roll back this deploy."
174
+ logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
175
+
176
+ unidentified_errors = []
177
+ filenames_with_sensitive_content = resources
178
+ .select(&:sensitive_template_content?)
179
+ .map { |r| File.basename(r.file_path) }
180
+
181
+ server_dry_run_validated_resource = resources
182
+ .select(&:server_dry_run_validated?)
183
+ .map { |r| File.basename(r.file_path) }
184
+
185
+ err.each_line do |line|
186
+ bad_files = find_bad_files_from_kubectl_output(line)
187
+ unless bad_files.present?
188
+ unidentified_errors << line
189
+ next
190
+ end
191
+
192
+ bad_files.each do |f|
193
+ err_msg = f[:err]
194
+ if filenames_with_sensitive_content.include?(f[:filename])
195
+ # Hide the error and template contents in case it has sensitive information
196
+ # we display full error messages as we assume there's no sensitive info leak after server-dry-run
197
+ err_msg = "SUPPRESSED FOR SECURITY" unless server_dry_run_validated_resource.include?(f[:filename])
198
+ record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: nil)
199
+ else
200
+ record_invalid_template(logger: logger, err: err_msg, filename: f[:filename], content: f[:content])
201
+ end
202
+ end
203
+ end
204
+ return unless unidentified_errors.any?
205
+
206
+ if (filenames_with_sensitive_content - server_dry_run_validated_resource).present?
207
+ warn_msg = "WARNING: There was an error applying some or all resources. The raw output may be sensitive and " \
208
+ "so cannot be displayed."
209
+ logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
210
+ else
211
+ heading = ColorizedString.new('Unidentified error(s):').red
212
+ msg = FormattedLogger.indent_four(unidentified_errors.join)
213
+ logger.summary.add_paragraph("#{heading}\n#{msg}")
214
+ end
215
+ end
216
+
217
+ def replace_or_create_resource(resource, force: false)
218
+ args = if force
219
+ ["replace", "--force", "--cascade", "-f", resource.file_path]
220
+ else
221
+ ["replace", "-f", resource.file_path]
222
+ end
223
+
224
+ _, err, status = kubectl.run(*args, log_failure: false, output_is_sensitive: resource.sensitive_template_content?,
225
+ raise_if_not_found: true, use_namespace: !resource.global?)
226
+
227
+ [err, status]
228
+ rescue Krane::Kubectl::ResourceNotFoundError
229
+ # it doesn't exist so we can't replace it, we try to create it
230
+ create_resource(resource)
231
+ end
232
+
233
+ def create_resource(resource)
234
+ out, err, status = kubectl.run("create", "-f", resource.file_path, log_failure: false,
235
+ output: 'json', output_is_sensitive: resource.sensitive_template_content?,
236
+ use_namespace: !resource.global?)
237
+
238
+ # For resources that rely on a generateName attribute, we get the `name` from the result of the call to `create`
239
+ # We must explicitly set this name value so that the `apply` step for pruning can run successfully
240
+ if status.success? && resource.uses_generate_name?
241
+ resource.use_generated_name(JSON.parse(out))
242
+ end
243
+
244
+ [err, status]
245
+ end
246
+
247
+ # Inspect the file referenced in the kubectl stderr
248
+ # to make it easier for developer to understand what's going on
249
+ def find_bad_files_from_kubectl_output(line)
250
+ # stderr often contains one or more lines like the following, from which we can extract the file path(s):
251
+ # Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
252
+
253
+ line.scan(%r{"(/\S+\.ya?ml\S*)"}).each_with_object([]) do |matches, bad_files|
254
+ matches.each do |path|
255
+ content = File.read(path) if File.file?(path)
256
+ bad_files << { filename: File.basename(path), err: line, content: content }
257
+ end
258
+ end
259
+ end
260
+
261
+ def kubectl
262
+ @kubectl ||= Kubectl.new(task_config: @task_config, log_failure_by_default: true)
263
+ end
264
+ end
265
+ end